Line data Source code
1 : /*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2 : |* *|
3 : |* "Fast" Instruction Selector for the X86 target *|
4 : |* *|
5 : |* Automatically generated file, do not edit! *|
6 : |* *|
7 : \*===----------------------------------------------------------------------===*/
8 :
9 :
10 : // FastEmit Immediate Predicate functions.
11 : static bool Predicate_i16immSExt8(int64_t Imm) {
12 : return isInt<8>(Imm);
13 : }
14 : static bool Predicate_i32immSExt8(int64_t Imm) {
15 : return isInt<8>(Imm);
16 : }
17 : static bool Predicate_i64immSExt8(int64_t Imm) {
18 : return isInt<8>(Imm);
19 : }
20 : static bool Predicate_i64immSExt32(int64_t Imm) {
21 : return isInt<32>(Imm);
22 : }
23 : static bool Predicate_AndMask64(int64_t Imm) {
24 :
25 : return isMask_64(Imm) && !isUInt<32>(Imm);
26 :
27 : }
28 : static bool Predicate_BTRMask64(int64_t Imm) {
29 :
30 : return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm);
31 :
32 : }
33 : static bool Predicate_BTCBTSMask64(int64_t Imm) {
34 :
35 : return !isInt<32>(Imm) && isPowerOf2_64(Imm);
36 :
37 : }
38 :
39 :
40 : // FastEmit functions for ISD::ABS.
41 :
42 0 : unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
43 0 : if (RetVT.SimpleTy != MVT::v16i8)
44 : return 0;
45 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
46 0 : return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
47 : }
48 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
49 0 : return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
50 : }
51 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
52 0 : return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
53 : }
54 : return 0;
55 : }
56 :
57 0 : unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
58 0 : if (RetVT.SimpleTy != MVT::v32i8)
59 : return 0;
60 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
61 0 : return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
62 : }
63 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
64 0 : return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
65 : }
66 : return 0;
67 : }
68 :
69 : unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
70 0 : if (RetVT.SimpleTy != MVT::v64i8)
71 : return 0;
72 0 : if ((Subtarget->hasBWI())) {
73 0 : return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
74 : }
75 : return 0;
76 : }
77 :
78 0 : unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
79 0 : if (RetVT.SimpleTy != MVT::v8i16)
80 : return 0;
81 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
82 0 : return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
83 : }
84 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
85 0 : return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
86 : }
87 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
88 0 : return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
89 : }
90 : return 0;
91 : }
92 :
93 0 : unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
94 0 : if (RetVT.SimpleTy != MVT::v16i16)
95 : return 0;
96 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
97 0 : return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
98 : }
99 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
100 0 : return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
101 : }
102 : return 0;
103 : }
104 :
105 : unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
106 0 : if (RetVT.SimpleTy != MVT::v32i16)
107 : return 0;
108 0 : if ((Subtarget->hasBWI())) {
109 0 : return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
110 : }
111 : return 0;
112 : }
113 :
114 0 : unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
115 0 : if (RetVT.SimpleTy != MVT::v4i32)
116 : return 0;
117 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
118 0 : return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
119 : }
120 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
121 0 : return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
122 : }
123 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
124 0 : return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
125 : }
126 : return 0;
127 : }
128 :
129 0 : unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
130 0 : if (RetVT.SimpleTy != MVT::v8i32)
131 : return 0;
132 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
133 0 : return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
134 : }
135 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
136 0 : return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
137 : }
138 : return 0;
139 : }
140 :
141 : unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
142 0 : if (RetVT.SimpleTy != MVT::v16i32)
143 : return 0;
144 0 : if ((Subtarget->hasAVX512())) {
145 0 : return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
146 : }
147 : return 0;
148 : }
149 :
150 0 : unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
151 0 : if (RetVT.SimpleTy != MVT::v2i64)
152 : return 0;
153 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154 0 : return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
155 : }
156 : return 0;
157 : }
158 :
159 0 : unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
160 0 : if (RetVT.SimpleTy != MVT::v4i64)
161 : return 0;
162 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
163 0 : return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
164 : }
165 : return 0;
166 : }
167 :
168 : unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
169 0 : if (RetVT.SimpleTy != MVT::v8i64)
170 : return 0;
171 0 : if ((Subtarget->hasAVX512())) {
172 0 : return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
173 : }
174 : return 0;
175 : }
176 :
177 0 : unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
178 0 : switch (VT.SimpleTy) {
179 0 : case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
180 0 : case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
181 0 : case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
182 0 : case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
183 0 : case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
184 0 : case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
185 0 : case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
186 0 : case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
187 0 : case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
188 0 : case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
189 0 : case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
190 0 : case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
191 : default: return 0;
192 : }
193 : }
194 :
195 : // FastEmit functions for ISD::ANY_EXTEND.
196 :
197 : unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
198 33 : if (RetVT.SimpleTy != MVT::i32)
199 : return 0;
200 33 : return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
201 : }
202 :
203 : unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
204 36 : switch (VT.SimpleTy) {
205 33 : case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
206 : default: return 0;
207 : }
208 : }
209 :
210 : // FastEmit functions for ISD::BITCAST.
211 :
212 23 : unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
213 23 : if (RetVT.SimpleTy != MVT::f32)
214 : return 0;
215 22 : if ((Subtarget->hasAVX512())) {
216 0 : return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill);
217 : }
218 11 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
219 11 : return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
220 : }
221 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
222 0 : return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
223 : }
224 : return 0;
225 : }
226 :
227 135 : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
228 270 : if ((Subtarget->hasAVX512())) {
229 0 : return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill);
230 : }
231 135 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
232 135 : return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
233 : }
234 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
235 0 : return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
236 : }
237 : return 0;
238 : }
239 :
240 : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
241 0 : if ((Subtarget->hasMMX())) {
242 0 : return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill);
243 : }
244 : return 0;
245 : }
246 :
247 138 : unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
248 138 : switch (RetVT.SimpleTy) {
249 135 : case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
250 0 : case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill);
251 : default: return 0;
252 : }
253 : }
254 :
255 26 : unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
256 26 : if (RetVT.SimpleTy != MVT::i32)
257 : return 0;
258 52 : if ((Subtarget->hasAVX512())) {
259 2 : return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
260 : }
261 24 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
262 22 : return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
263 : }
264 2 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
265 2 : return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
266 : }
267 : return 0;
268 : }
269 :
270 136 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
271 272 : if ((Subtarget->hasAVX512())) {
272 0 : return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
273 : }
274 136 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
275 135 : return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
276 : }
277 1 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
278 1 : return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
279 : }
280 : return 0;
281 : }
282 :
283 0 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
284 0 : if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
285 0 : return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
286 : }
287 : return 0;
288 : }
289 :
290 136 : unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
291 136 : switch (RetVT.SimpleTy) {
292 136 : case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
293 0 : case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill);
294 : default: return 0;
295 : }
296 : }
297 :
298 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
299 0 : if ((Subtarget->hasMMX())) {
300 0 : return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill);
301 : }
302 : return 0;
303 : }
304 :
305 0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
306 0 : if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
307 0 : return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill);
308 : }
309 : return 0;
310 : }
311 :
312 0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
313 0 : switch (RetVT.SimpleTy) {
314 0 : case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill);
315 0 : case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill);
316 : default: return 0;
317 : }
318 : }
319 :
320 4267 : unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
321 4267 : switch (VT.SimpleTy) {
322 23 : case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
323 138 : case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
324 26 : case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill);
325 136 : case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
326 0 : case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
327 : default: return 0;
328 : }
329 : }
330 :
331 : // FastEmit functions for ISD::BRIND.
332 :
333 : unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
334 0 : if (RetVT.SimpleTy != MVT::isVoid)
335 : return 0;
336 0 : if ((!Subtarget->is64Bit())) {
337 0 : return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill);
338 : }
339 : return 0;
340 : }
341 :
342 : unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
343 0 : if (RetVT.SimpleTy != MVT::isVoid)
344 : return 0;
345 0 : if ((!Subtarget->is64Bit())) {
346 0 : return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill);
347 : }
348 : return 0;
349 : }
350 :
351 : unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
352 0 : if (RetVT.SimpleTy != MVT::isVoid)
353 : return 0;
354 0 : if ((Subtarget->is64Bit())) {
355 0 : return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill);
356 : }
357 : return 0;
358 : }
359 :
360 0 : unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
361 0 : switch (VT.SimpleTy) {
362 0 : case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
363 0 : case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
364 0 : case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
365 : default: return 0;
366 : }
367 : }
368 :
369 : // FastEmit functions for ISD::BSWAP.
370 :
371 : unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
372 0 : if (RetVT.SimpleTy != MVT::i32)
373 : return 0;
374 0 : return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill);
375 : }
376 :
377 : unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
378 0 : if (RetVT.SimpleTy != MVT::i64)
379 : return 0;
380 0 : return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill);
381 : }
382 :
383 0 : unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
384 0 : switch (VT.SimpleTy) {
385 0 : case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
386 0 : case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
387 : default: return 0;
388 : }
389 : }
390 :
391 : // FastEmit functions for ISD::CTLZ.
392 :
393 : unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
394 0 : if (RetVT.SimpleTy != MVT::i16)
395 : return 0;
396 0 : if ((Subtarget->hasLZCNT())) {
397 0 : return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
398 : }
399 : return 0;
400 : }
401 :
402 : unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
403 0 : if (RetVT.SimpleTy != MVT::i32)
404 : return 0;
405 0 : if ((Subtarget->hasLZCNT())) {
406 0 : return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
407 : }
408 : return 0;
409 : }
410 :
411 : unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
412 0 : if (RetVT.SimpleTy != MVT::i64)
413 : return 0;
414 0 : if ((Subtarget->hasLZCNT())) {
415 0 : return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
416 : }
417 : return 0;
418 : }
419 :
420 0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
421 0 : if (RetVT.SimpleTy != MVT::v4i32)
422 : return 0;
423 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
424 0 : return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
425 : }
426 : return 0;
427 : }
428 :
429 0 : unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
430 0 : if (RetVT.SimpleTy != MVT::v8i32)
431 : return 0;
432 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
433 0 : return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
434 : }
435 : return 0;
436 : }
437 :
438 : unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
439 0 : if (RetVT.SimpleTy != MVT::v16i32)
440 : return 0;
441 0 : if ((Subtarget->hasCDI())) {
442 0 : return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
443 : }
444 : return 0;
445 : }
446 :
447 0 : unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
448 0 : if (RetVT.SimpleTy != MVT::v2i64)
449 : return 0;
450 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
451 0 : return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
452 : }
453 : return 0;
454 : }
455 :
456 0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
457 0 : if (RetVT.SimpleTy != MVT::v4i64)
458 : return 0;
459 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
460 0 : return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
461 : }
462 : return 0;
463 : }
464 :
465 : unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
466 0 : if (RetVT.SimpleTy != MVT::v8i64)
467 : return 0;
468 0 : if ((Subtarget->hasCDI())) {
469 0 : return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
470 : }
471 : return 0;
472 : }
473 :
474 0 : unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
475 0 : switch (VT.SimpleTy) {
476 0 : case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
477 0 : case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
478 0 : case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
479 0 : case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
480 0 : case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
481 0 : case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
482 0 : case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
483 0 : case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
484 0 : case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
485 : default: return 0;
486 : }
487 : }
488 :
489 : // FastEmit functions for ISD::CTPOP.
490 :
491 : unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
492 0 : if (RetVT.SimpleTy != MVT::i16)
493 : return 0;
494 0 : if ((Subtarget->hasPOPCNT())) {
495 0 : return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
496 : }
497 : return 0;
498 : }
499 :
500 : unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
501 0 : if (RetVT.SimpleTy != MVT::i32)
502 : return 0;
503 0 : if ((Subtarget->hasPOPCNT())) {
504 0 : return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
505 : }
506 : return 0;
507 : }
508 :
509 : unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
510 0 : if (RetVT.SimpleTy != MVT::i64)
511 : return 0;
512 0 : if ((Subtarget->hasPOPCNT())) {
513 0 : return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
514 : }
515 : return 0;
516 : }
517 :
518 0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
519 0 : if (RetVT.SimpleTy != MVT::v16i8)
520 : return 0;
521 0 : if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
522 0 : return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
523 : }
524 : return 0;
525 : }
526 :
527 0 : unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
528 0 : if (RetVT.SimpleTy != MVT::v32i8)
529 : return 0;
530 0 : if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
531 0 : return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
532 : }
533 : return 0;
534 : }
535 :
536 : unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
537 0 : if (RetVT.SimpleTy != MVT::v64i8)
538 : return 0;
539 0 : if ((Subtarget->hasBITALG())) {
540 0 : return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
541 : }
542 : return 0;
543 : }
544 :
545 0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
546 0 : if (RetVT.SimpleTy != MVT::v8i16)
547 : return 0;
548 0 : if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
549 0 : return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
550 : }
551 : return 0;
552 : }
553 :
554 0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
555 0 : if (RetVT.SimpleTy != MVT::v16i16)
556 : return 0;
557 0 : if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
558 0 : return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
559 : }
560 : return 0;
561 : }
562 :
563 : unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
564 0 : if (RetVT.SimpleTy != MVT::v32i16)
565 : return 0;
566 0 : if ((Subtarget->hasBITALG())) {
567 0 : return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
568 : }
569 : return 0;
570 : }
571 :
572 0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
573 0 : if (RetVT.SimpleTy != MVT::v4i32)
574 : return 0;
575 0 : if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
576 0 : return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
577 : }
578 : return 0;
579 : }
580 :
581 0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
582 0 : if (RetVT.SimpleTy != MVT::v8i32)
583 : return 0;
584 0 : if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
585 0 : return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
586 : }
587 : return 0;
588 : }
589 :
590 : unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
591 0 : if (RetVT.SimpleTy != MVT::v16i32)
592 : return 0;
593 0 : if ((Subtarget->hasVPOPCNTDQ())) {
594 0 : return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
595 : }
596 : return 0;
597 : }
598 :
599 0 : unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
600 0 : if (RetVT.SimpleTy != MVT::v2i64)
601 : return 0;
602 0 : if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
603 0 : return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
604 : }
605 : return 0;
606 : }
607 :
608 0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
609 0 : if (RetVT.SimpleTy != MVT::v4i64)
610 : return 0;
611 0 : if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
612 0 : return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
613 : }
614 : return 0;
615 : }
616 :
617 : unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
618 0 : if (RetVT.SimpleTy != MVT::v8i64)
619 : return 0;
620 0 : if ((Subtarget->hasVPOPCNTDQ())) {
621 0 : return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
622 : }
623 : return 0;
624 : }
625 :
626 0 : unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
627 0 : switch (VT.SimpleTy) {
628 0 : case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill);
629 0 : case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill);
630 0 : case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill);
631 0 : case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
632 0 : case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
633 0 : case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
634 0 : case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
635 0 : case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
636 0 : case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
637 0 : case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
638 0 : case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
639 0 : case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
640 0 : case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
641 0 : case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
642 0 : case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
643 : default: return 0;
644 : }
645 : }
646 :
647 : // FastEmit functions for ISD::CTTZ.
648 :
649 : unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
650 0 : if (RetVT.SimpleTy != MVT::i16)
651 : return 0;
652 0 : if ((Subtarget->hasBMI())) {
653 0 : return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
654 : }
655 : return 0;
656 : }
657 :
658 : unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
659 0 : if (RetVT.SimpleTy != MVT::i32)
660 : return 0;
661 0 : if ((Subtarget->hasBMI())) {
662 0 : return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
663 : }
664 : return 0;
665 : }
666 :
667 : unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
668 0 : if (RetVT.SimpleTy != MVT::i64)
669 : return 0;
670 0 : if ((Subtarget->hasBMI())) {
671 0 : return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
672 : }
673 : return 0;
674 : }
675 :
676 0 : unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
677 0 : switch (VT.SimpleTy) {
678 0 : case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
679 0 : case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
680 0 : case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
681 : default: return 0;
682 : }
683 : }
684 :
685 : // FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
686 :
687 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
688 0 : if (RetVT.SimpleTy != MVT::i16)
689 : return 0;
690 0 : return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill);
691 : }
692 :
693 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
694 0 : if (RetVT.SimpleTy != MVT::i32)
695 : return 0;
696 0 : return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill);
697 : }
698 :
699 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
700 0 : if (RetVT.SimpleTy != MVT::i64)
701 : return 0;
702 0 : return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill);
703 : }
704 :
705 0 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
706 0 : switch (VT.SimpleTy) {
707 0 : case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill);
708 0 : case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill);
709 0 : case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill);
710 : default: return 0;
711 : }
712 : }
713 :
714 : // FastEmit functions for ISD::FABS.
715 :
716 : unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
717 0 : if (RetVT.SimpleTy != MVT::f32)
718 : return 0;
719 0 : if ((!Subtarget->hasSSE1())) {
720 0 : return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
721 : }
722 : return 0;
723 : }
724 :
725 : unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
726 0 : if (RetVT.SimpleTy != MVT::f64)
727 : return 0;
728 0 : if ((!Subtarget->hasSSE2())) {
729 0 : return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
730 : }
731 : return 0;
732 : }
733 :
734 : unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
735 0 : if (RetVT.SimpleTy != MVT::f80)
736 : return 0;
737 0 : return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
738 : }
739 :
740 0 : unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
741 0 : switch (VT.SimpleTy) {
742 0 : case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
743 0 : case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
744 0 : case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill);
745 : default: return 0;
746 : }
747 : }
748 :
749 : // FastEmit functions for ISD::FCOS.
750 :
751 : unsigned fastEmit_ISD_FCOS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
752 0 : if (RetVT.SimpleTy != MVT::f32)
753 : return 0;
754 0 : if ((!Subtarget->hasSSE1())) {
755 0 : return fastEmitInst_r(X86::COS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
756 : }
757 : return 0;
758 : }
759 :
760 : unsigned fastEmit_ISD_FCOS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
761 0 : if (RetVT.SimpleTy != MVT::f64)
762 : return 0;
763 0 : if ((!Subtarget->hasSSE2())) {
764 0 : return fastEmitInst_r(X86::COS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
765 : }
766 : return 0;
767 : }
768 :
769 : unsigned fastEmit_ISD_FCOS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
770 0 : if (RetVT.SimpleTy != MVT::f80)
771 : return 0;
772 0 : return fastEmitInst_r(X86::COS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
773 : }
774 :
775 0 : unsigned fastEmit_ISD_FCOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
776 0 : switch (VT.SimpleTy) {
777 0 : case MVT::f32: return fastEmit_ISD_FCOS_MVT_f32_r(RetVT, Op0, Op0IsKill);
778 0 : case MVT::f64: return fastEmit_ISD_FCOS_MVT_f64_r(RetVT, Op0, Op0IsKill);
779 0 : case MVT::f80: return fastEmit_ISD_FCOS_MVT_f80_r(RetVT, Op0, Op0IsKill);
780 : default: return 0;
781 : }
782 : }
783 :
784 : // FastEmit functions for ISD::FNEG.
785 :
786 : unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
787 11 : if (RetVT.SimpleTy != MVT::f32)
788 : return 0;
789 22 : if ((!Subtarget->hasSSE1())) {
790 0 : return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
791 : }
792 : return 0;
793 : }
794 :
795 : unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
796 135 : if (RetVT.SimpleTy != MVT::f64)
797 : return 0;
798 270 : if ((!Subtarget->hasSSE2())) {
799 0 : return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
800 : }
801 : return 0;
802 : }
803 :
804 : unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
805 2 : if (RetVT.SimpleTy != MVT::f80)
806 : return 0;
807 2 : return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
808 : }
809 :
810 188 : unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
811 188 : switch (VT.SimpleTy) {
812 11 : case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
813 135 : case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
814 2 : case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill);
815 : default: return 0;
816 : }
817 : }
818 :
819 : // FastEmit functions for ISD::FP_EXTEND.
820 :
821 0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
822 0 : if (RetVT.SimpleTy != MVT::f64)
823 : return 0;
824 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
825 0 : return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
826 : }
827 : return 0;
828 : }
829 :
830 0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
831 0 : if (RetVT.SimpleTy != MVT::v4f64)
832 : return 0;
833 0 : if ((Subtarget->hasVLX())) {
834 0 : return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
835 : }
836 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
837 0 : return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
838 : }
839 : return 0;
840 : }
841 :
842 : unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
843 0 : if (RetVT.SimpleTy != MVT::v8f64)
844 : return 0;
845 0 : if ((Subtarget->hasAVX512())) {
846 0 : return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
847 : }
848 : return 0;
849 : }
850 :
851 0 : unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
852 0 : switch (VT.SimpleTy) {
853 0 : case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
854 0 : case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
855 0 : case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
856 : default: return 0;
857 : }
858 : }
859 :
860 : // FastEmit functions for ISD::FP_ROUND.
861 :
862 0 : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
863 0 : if (RetVT.SimpleTy != MVT::f32)
864 : return 0;
865 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
866 0 : return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
867 : }
868 : return 0;
869 : }
870 :
871 0 : unsigned fastEmit_ISD_FP_ROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
872 0 : if (RetVT.SimpleTy != MVT::v4f32)
873 : return 0;
874 0 : if ((Subtarget->hasVLX())) {
875 0 : return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
876 : }
877 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
878 0 : return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
879 : }
880 : return 0;
881 : }
882 :
883 : unsigned fastEmit_ISD_FP_ROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
884 0 : if (RetVT.SimpleTy != MVT::v8f32)
885 : return 0;
886 0 : if ((Subtarget->hasAVX512())) {
887 0 : return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
888 : }
889 : return 0;
890 : }
891 :
892 0 : unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
893 0 : switch (VT.SimpleTy) {
894 0 : case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
895 0 : case MVT::v4f64: return fastEmit_ISD_FP_ROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
896 0 : case MVT::v8f64: return fastEmit_ISD_FP_ROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
897 : default: return 0;
898 : }
899 : }
900 :
901 : // FastEmit functions for ISD::FP_TO_SINT.
902 :
903 9 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
904 18 : if ((Subtarget->hasAVX512())) {
905 0 : return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
906 : }
907 9 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
908 9 : return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
909 : }
910 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
911 0 : return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
912 : }
913 : return 0;
914 : }
915 :
916 4 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
917 8 : if ((Subtarget->hasAVX512())) {
918 0 : return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
919 : }
920 4 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
921 4 : return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
922 : }
923 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
924 0 : return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
925 : }
926 : return 0;
927 : }
928 :
929 13 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
930 13 : switch (RetVT.SimpleTy) {
931 9 : case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
932 4 : case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
933 : default: return 0;
934 : }
935 : }
936 :
937 69 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
938 138 : if ((Subtarget->hasAVX512())) {
939 0 : return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
940 : }
941 69 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
942 69 : return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
943 : }
944 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
945 0 : return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
946 : }
947 : return 0;
948 : }
949 :
950 34 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
951 68 : if ((Subtarget->hasAVX512())) {
952 0 : return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
953 : }
954 34 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
955 34 : return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
956 : }
957 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
958 0 : return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
959 : }
960 : return 0;
961 : }
962 :
963 103 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
964 103 : switch (RetVT.SimpleTy) {
965 69 : case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
966 34 : case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
967 : default: return 0;
968 : }
969 : }
970 :
971 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
972 0 : if ((Subtarget->hasVLX())) {
973 0 : return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
974 : }
975 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
976 0 : return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
977 : }
978 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
979 0 : return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
980 : }
981 : return 0;
982 : }
983 :
984 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
985 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
986 0 : return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
987 : }
988 : return 0;
989 : }
990 :
991 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
992 0 : switch (RetVT.SimpleTy) {
993 0 : case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
994 0 : case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
995 : default: return 0;
996 : }
997 : }
998 :
999 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1000 0 : if ((Subtarget->hasVLX())) {
1001 0 : return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1002 : }
1003 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1004 0 : return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1005 : }
1006 : return 0;
1007 : }
1008 :
1009 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1010 0 : if ((Subtarget->hasDQI())) {
1011 0 : return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1012 : }
1013 : return 0;
1014 : }
1015 :
1016 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1017 0 : switch (RetVT.SimpleTy) {
1018 0 : case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
1019 0 : case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
1020 : default: return 0;
1021 : }
1022 : }
1023 :
1024 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1025 0 : if (RetVT.SimpleTy != MVT::v16i32)
1026 : return 0;
1027 0 : if ((Subtarget->hasAVX512())) {
1028 0 : return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1029 : }
1030 : return 0;
1031 : }
1032 :
1033 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1034 0 : if (RetVT.SimpleTy != MVT::v2i64)
1035 : return 0;
1036 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1037 0 : return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1038 : }
1039 : return 0;
1040 : }
1041 :
1042 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1043 0 : if ((Subtarget->hasVLX())) {
1044 0 : return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1045 : }
1046 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1047 0 : return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
1048 : }
1049 : return 0;
1050 : }
1051 :
1052 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1053 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1054 0 : return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1055 : }
1056 : return 0;
1057 : }
1058 :
1059 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1060 0 : switch (RetVT.SimpleTy) {
1061 0 : case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
1062 0 : case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
1063 : default: return 0;
1064 : }
1065 : }
1066 :
1067 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1068 0 : if ((Subtarget->hasAVX512())) {
1069 0 : return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
1070 : }
1071 : return 0;
1072 : }
1073 :
1074 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1075 0 : if ((Subtarget->hasDQI())) {
1076 0 : return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1077 : }
1078 : return 0;
1079 : }
1080 :
1081 0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1082 0 : switch (RetVT.SimpleTy) {
1083 0 : case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
1084 0 : case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
1085 : default: return 0;
1086 : }
1087 : }
1088 :
1089 120 : unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1090 120 : switch (VT.SimpleTy) {
1091 13 : case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1092 103 : case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1093 0 : case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1094 0 : case MVT::v8f32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
1095 0 : case MVT::v16f32: return fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
1096 0 : case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1097 0 : case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
1098 0 : case MVT::v8f64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
1099 : default: return 0;
1100 : }
1101 : }
1102 :
1103 : // FastEmit functions for ISD::FP_TO_UINT.
1104 :
1105 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1106 0 : if ((Subtarget->hasAVX512())) {
1107 0 : return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
1108 : }
1109 : return 0;
1110 : }
1111 :
1112 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1113 0 : if ((Subtarget->hasAVX512())) {
1114 0 : return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1115 : }
1116 : return 0;
1117 : }
1118 :
1119 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1120 0 : switch (RetVT.SimpleTy) {
1121 0 : case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
1122 0 : case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
1123 : default: return 0;
1124 : }
1125 : }
1126 :
1127 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1128 0 : if ((Subtarget->hasAVX512())) {
1129 0 : return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
1130 : }
1131 : return 0;
1132 : }
1133 :
1134 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1135 0 : if ((Subtarget->hasAVX512())) {
1136 0 : return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1137 : }
1138 : return 0;
1139 : }
1140 :
1141 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1142 0 : switch (RetVT.SimpleTy) {
1143 0 : case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
1144 0 : case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
1145 : default: return 0;
1146 : }
1147 : }
1148 :
1149 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1150 0 : if ((Subtarget->hasVLX())) {
1151 0 : return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1152 : }
1153 : return 0;
1154 : }
1155 :
1156 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1157 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1158 0 : return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1159 : }
1160 : return 0;
1161 : }
1162 :
1163 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1164 0 : switch (RetVT.SimpleTy) {
1165 0 : case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
1166 0 : case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
1167 : default: return 0;
1168 : }
1169 : }
1170 :
1171 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1172 0 : if ((Subtarget->hasVLX())) {
1173 0 : return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1174 : }
1175 : return 0;
1176 : }
1177 :
1178 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1179 0 : if ((Subtarget->hasDQI())) {
1180 0 : return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1181 : }
1182 : return 0;
1183 : }
1184 :
1185 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1186 0 : switch (RetVT.SimpleTy) {
1187 0 : case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
1188 0 : case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
1189 : default: return 0;
1190 : }
1191 : }
1192 :
1193 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1194 0 : if (RetVT.SimpleTy != MVT::v16i32)
1195 : return 0;
1196 0 : if ((Subtarget->hasAVX512())) {
1197 0 : return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1198 : }
1199 : return 0;
1200 : }
1201 :
1202 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1203 0 : if (RetVT.SimpleTy != MVT::v2i64)
1204 : return 0;
1205 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1206 0 : return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1207 : }
1208 : return 0;
1209 : }
1210 :
1211 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1212 0 : if ((Subtarget->hasVLX())) {
1213 0 : return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1214 : }
1215 : return 0;
1216 : }
1217 :
1218 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1219 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1220 0 : return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1221 : }
1222 : return 0;
1223 : }
1224 :
1225 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1226 0 : switch (RetVT.SimpleTy) {
1227 0 : case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
1228 0 : case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
1229 : default: return 0;
1230 : }
1231 : }
1232 :
1233 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1234 0 : if ((Subtarget->hasAVX512())) {
1235 0 : return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
1236 : }
1237 : return 0;
1238 : }
1239 :
1240 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1241 0 : if ((Subtarget->hasDQI())) {
1242 0 : return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1243 : }
1244 : return 0;
1245 : }
1246 :
1247 0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1248 0 : switch (RetVT.SimpleTy) {
1249 0 : case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
1250 0 : case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
1251 : default: return 0;
1252 : }
1253 : }
1254 :
1255 0 : unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1256 0 : switch (VT.SimpleTy) {
1257 0 : case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1258 0 : case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1259 0 : case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1260 0 : case MVT::v8f32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
1261 0 : case MVT::v16f32: return fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
1262 0 : case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1263 0 : case MVT::v4f64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
1264 0 : case MVT::v8f64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
1265 : default: return 0;
1266 : }
1267 : }
1268 :
1269 : // FastEmit functions for ISD::FSIN.
1270 :
1271 : unsigned fastEmit_ISD_FSIN_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1272 0 : if (RetVT.SimpleTy != MVT::f32)
1273 : return 0;
1274 0 : if ((!Subtarget->hasSSE1())) {
1275 0 : return fastEmitInst_r(X86::SIN_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
1276 : }
1277 : return 0;
1278 : }
1279 :
1280 : unsigned fastEmit_ISD_FSIN_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1281 0 : if (RetVT.SimpleTy != MVT::f64)
1282 : return 0;
1283 0 : if ((!Subtarget->hasSSE2())) {
1284 0 : return fastEmitInst_r(X86::SIN_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
1285 : }
1286 : return 0;
1287 : }
1288 :
1289 : unsigned fastEmit_ISD_FSIN_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1290 0 : if (RetVT.SimpleTy != MVT::f80)
1291 : return 0;
1292 0 : return fastEmitInst_r(X86::SIN_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
1293 : }
1294 :
1295 0 : unsigned fastEmit_ISD_FSIN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1296 0 : switch (VT.SimpleTy) {
1297 0 : case MVT::f32: return fastEmit_ISD_FSIN_MVT_f32_r(RetVT, Op0, Op0IsKill);
1298 0 : case MVT::f64: return fastEmit_ISD_FSIN_MVT_f64_r(RetVT, Op0, Op0IsKill);
1299 0 : case MVT::f80: return fastEmit_ISD_FSIN_MVT_f80_r(RetVT, Op0, Op0IsKill);
1300 : default: return 0;
1301 : }
1302 : }
1303 :
1304 : // FastEmit functions for ISD::FSQRT.
1305 :
1306 0 : unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1307 0 : if (RetVT.SimpleTy != MVT::f32)
1308 : return 0;
1309 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1310 0 : return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
1311 : }
1312 0 : if ((!Subtarget->hasSSE1())) {
1313 0 : return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
1314 : }
1315 : return 0;
1316 : }
1317 :
1318 0 : unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1319 0 : if (RetVT.SimpleTy != MVT::f64)
1320 : return 0;
1321 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1322 0 : return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
1323 : }
1324 0 : if ((!Subtarget->hasSSE2())) {
1325 0 : return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
1326 : }
1327 : return 0;
1328 : }
1329 :
1330 : unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1331 0 : if (RetVT.SimpleTy != MVT::f80)
1332 : return 0;
1333 0 : return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
1334 : }
1335 :
1336 0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1337 0 : if (RetVT.SimpleTy != MVT::v4f32)
1338 : return 0;
1339 0 : if ((Subtarget->hasVLX())) {
1340 0 : return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
1341 : }
1342 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1343 0 : return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
1344 : }
1345 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1346 0 : return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
1347 : }
1348 : return 0;
1349 : }
1350 :
1351 0 : unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1352 0 : if (RetVT.SimpleTy != MVT::v8f32)
1353 : return 0;
1354 0 : if ((Subtarget->hasVLX())) {
1355 0 : return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
1356 : }
1357 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1358 0 : return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
1359 : }
1360 : return 0;
1361 : }
1362 :
1363 : unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1364 0 : if (RetVT.SimpleTy != MVT::v16f32)
1365 : return 0;
1366 0 : if ((Subtarget->hasAVX512())) {
1367 0 : return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
1368 : }
1369 : return 0;
1370 : }
1371 :
1372 0 : unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1373 0 : if (RetVT.SimpleTy != MVT::v2f64)
1374 : return 0;
1375 0 : if ((Subtarget->hasVLX())) {
1376 0 : return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
1377 : }
1378 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1379 0 : return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
1380 : }
1381 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1382 0 : return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
1383 : }
1384 : return 0;
1385 : }
1386 :
1387 0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1388 0 : if (RetVT.SimpleTy != MVT::v4f64)
1389 : return 0;
1390 0 : if ((Subtarget->hasVLX())) {
1391 0 : return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
1392 : }
1393 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1394 0 : return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
1395 : }
1396 : return 0;
1397 : }
1398 :
1399 : unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1400 0 : if (RetVT.SimpleTy != MVT::v8f64)
1401 : return 0;
1402 0 : if ((Subtarget->hasAVX512())) {
1403 0 : return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
1404 : }
1405 : return 0;
1406 : }
1407 :
1408 0 : unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1409 0 : switch (VT.SimpleTy) {
1410 0 : case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1411 0 : case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1412 0 : case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
1413 0 : case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1414 0 : case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
1415 0 : case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
1416 0 : case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1417 0 : case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
1418 0 : case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
1419 : default: return 0;
1420 : }
1421 : }
1422 :
1423 : // FastEmit functions for ISD::SCALAR_TO_VECTOR.
1424 :
1425 1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1426 2 : if ((Subtarget->hasAVX512())) {
1427 0 : return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
1428 : }
1429 1 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1430 0 : return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1431 : }
1432 1 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1433 1 : return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1434 : }
1435 : return 0;
1436 : }
1437 :
1438 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
1439 0 : if ((Subtarget->hasMMX())) {
1440 0 : return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
1441 : }
1442 : return 0;
1443 : }
1444 :
1445 1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1446 1 : switch (RetVT.SimpleTy) {
1447 1 : case MVT::v4i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
1448 0 : case MVT::x86mmx: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(Op0, Op0IsKill);
1449 : default: return 0;
1450 : }
1451 : }
1452 :
1453 0 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1454 0 : if (RetVT.SimpleTy != MVT::v2i64)
1455 : return 0;
1456 0 : if ((Subtarget->hasAVX512())) {
1457 0 : return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
1458 : }
1459 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1460 0 : return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1461 : }
1462 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1463 0 : return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1464 : }
1465 : return 0;
1466 : }
1467 :
1468 1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1469 1 : switch (VT.SimpleTy) {
1470 1 : case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill);
1471 0 : case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill);
1472 : default: return 0;
1473 : }
1474 : }
1475 :
1476 : // FastEmit functions for ISD::SIGN_EXTEND.
1477 :
1478 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1479 26432 : return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
1480 : }
1481 :
1482 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1483 107 : return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill);
1484 : }
1485 :
1486 26550 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1487 26550 : switch (RetVT.SimpleTy) {
1488 26432 : case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill);
1489 107 : case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill);
1490 : default: return 0;
1491 : }
1492 : }
1493 :
1494 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1495 645 : return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
1496 : }
1497 :
1498 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1499 32 : return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill);
1500 : }
1501 :
1502 677 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1503 677 : switch (RetVT.SimpleTy) {
1504 645 : case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill);
1505 32 : case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill);
1506 : default: return 0;
1507 : }
1508 : }
1509 :
1510 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1511 29063 : if (RetVT.SimpleTy != MVT::i64)
1512 : return 0;
1513 29063 : if ((Subtarget->is64Bit())) {
1514 29063 : return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill);
1515 : }
1516 : return 0;
1517 : }
1518 :
1519 28 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1520 28 : if (RetVT.SimpleTy != MVT::v2i64)
1521 : return 0;
1522 28 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1523 28 : return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1524 : }
1525 : return 0;
1526 : }
1527 :
1528 30 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1529 30 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1530 30 : return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1531 : }
1532 : return 0;
1533 : }
1534 :
1535 0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1536 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1537 0 : return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1538 : }
1539 : return 0;
1540 : }
1541 :
1542 30 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1543 30 : switch (RetVT.SimpleTy) {
1544 30 : case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill);
1545 0 : case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill);
1546 : default: return 0;
1547 : }
1548 : }
1549 :
1550 6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1551 6 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1552 6 : return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1553 : }
1554 : return 0;
1555 : }
1556 :
1557 0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1558 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1559 0 : return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1560 : }
1561 : return 0;
1562 : }
1563 :
1564 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1565 0 : if ((Subtarget->hasDQI())) {
1566 0 : return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1567 : }
1568 : return 0;
1569 : }
1570 :
1571 6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1572 6 : switch (RetVT.SimpleTy) {
1573 6 : case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill);
1574 0 : case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill);
1575 0 : case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
1576 : default: return 0;
1577 : }
1578 : }
1579 :
1580 6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1581 6 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1582 6 : return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1583 : }
1584 : return 0;
1585 : }
1586 :
1587 0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
1588 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1589 0 : return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1590 : }
1591 : return 0;
1592 : }
1593 :
1594 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
1595 0 : if ((Subtarget->hasDQI())) {
1596 0 : return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1597 : }
1598 : return 0;
1599 : }
1600 :
1601 6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1602 6 : switch (RetVT.SimpleTy) {
1603 6 : case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill);
1604 0 : case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill);
1605 0 : case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
1606 : default: return 0;
1607 : }
1608 : }
1609 :
1610 0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
1611 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1612 0 : return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1613 : }
1614 : return 0;
1615 : }
1616 :
1617 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
1618 0 : if ((Subtarget->hasBWI())) {
1619 0 : return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1620 : }
1621 : return 0;
1622 : }
1623 :
1624 0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1625 0 : switch (RetVT.SimpleTy) {
1626 0 : case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill);
1627 0 : case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill);
1628 : default: return 0;
1629 : }
1630 : }
1631 :
1632 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1633 0 : if (RetVT.SimpleTy != MVT::v64i8)
1634 : return 0;
1635 0 : if ((Subtarget->hasBWI())) {
1636 0 : return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1637 : }
1638 : return 0;
1639 : }
1640 :
1641 56366 : unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1642 56366 : switch (VT.SimpleTy) {
1643 26550 : case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
1644 677 : case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
1645 29063 : case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill);
1646 28 : case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill);
1647 30 : case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill);
1648 6 : case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
1649 6 : case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
1650 0 : case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill);
1651 0 : case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill);
1652 : default: return 0;
1653 : }
1654 : }
1655 :
1656 : // FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1657 :
1658 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1659 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1660 0 : return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
1661 : }
1662 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1663 0 : return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
1664 : }
1665 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1666 0 : return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1667 : }
1668 : return 0;
1669 : }
1670 :
1671 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1672 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1673 0 : return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1674 : }
1675 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1676 0 : return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1677 : }
1678 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1679 0 : return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1680 : }
1681 : return 0;
1682 : }
1683 :
1684 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1685 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1686 0 : return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1687 : }
1688 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1689 0 : return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1690 : }
1691 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1692 0 : return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1693 : }
1694 : return 0;
1695 : }
1696 :
1697 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1698 0 : switch (RetVT.SimpleTy) {
1699 0 : case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
1700 0 : case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
1701 0 : case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
1702 : default: return 0;
1703 : }
1704 : }
1705 :
1706 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1707 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1708 0 : return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1709 : }
1710 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1711 0 : return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1712 : }
1713 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1714 0 : return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1715 : }
1716 : return 0;
1717 : }
1718 :
1719 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1720 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1721 0 : return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1722 : }
1723 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1724 0 : return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1725 : }
1726 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1727 0 : return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1728 : }
1729 : return 0;
1730 : }
1731 :
1732 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1733 0 : switch (RetVT.SimpleTy) {
1734 0 : case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
1735 0 : case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
1736 : default: return 0;
1737 : }
1738 : }
1739 :
1740 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1741 0 : if (RetVT.SimpleTy != MVT::v2i64)
1742 : return 0;
1743 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1744 0 : return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1745 : }
1746 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1747 0 : return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1748 : }
1749 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1750 0 : return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1751 : }
1752 : return 0;
1753 : }
1754 :
1755 0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1756 0 : switch (VT.SimpleTy) {
1757 0 : case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1758 0 : case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1759 0 : case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1760 : default: return 0;
1761 : }
1762 : }
1763 :
1764 : // FastEmit functions for ISD::SINT_TO_FP.
1765 :
1766 46 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
1767 92 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1768 40 : return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
1769 : }
1770 : return 0;
1771 : }
1772 :
1773 972 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1774 1944 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1775 964 : return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
1776 : }
1777 : return 0;
1778 : }
1779 :
1780 1018 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1781 1018 : switch (RetVT.SimpleTy) {
1782 46 : case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
1783 972 : case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
1784 : default: return 0;
1785 : }
1786 : }
1787 :
1788 9 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
1789 18 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1790 3 : return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
1791 : }
1792 : return 0;
1793 : }
1794 :
1795 48 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1796 96 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1797 42 : return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
1798 : }
1799 : return 0;
1800 : }
1801 :
1802 73 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1803 73 : switch (RetVT.SimpleTy) {
1804 9 : case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
1805 48 : case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
1806 : default: return 0;
1807 : }
1808 : }
1809 :
1810 6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1811 6 : if ((Subtarget->hasVLX())) {
1812 2 : return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1813 : }
1814 8 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1815 2 : return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
1816 : }
1817 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1818 2 : return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
1819 : }
1820 : return 0;
1821 : }
1822 :
1823 2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
1824 2 : if ((Subtarget->hasVLX())) {
1825 0 : return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1826 : }
1827 4 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1828 2 : return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1829 : }
1830 : return 0;
1831 : }
1832 :
1833 8 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1834 8 : switch (RetVT.SimpleTy) {
1835 6 : case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
1836 2 : case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
1837 : default: return 0;
1838 : }
1839 : }
1840 :
1841 2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
1842 2 : if ((Subtarget->hasVLX())) {
1843 0 : return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1844 : }
1845 4 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1846 2 : return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1847 : }
1848 : return 0;
1849 : }
1850 :
1851 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
1852 0 : if ((Subtarget->hasAVX512())) {
1853 0 : return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1854 : }
1855 : return 0;
1856 : }
1857 :
1858 2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1859 2 : switch (RetVT.SimpleTy) {
1860 2 : case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
1861 0 : case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
1862 : default: return 0;
1863 : }
1864 : }
1865 :
1866 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1867 0 : if (RetVT.SimpleTy != MVT::v16f32)
1868 : return 0;
1869 0 : if ((Subtarget->hasAVX512())) {
1870 0 : return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1871 : }
1872 : return 0;
1873 : }
1874 :
1875 0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1876 0 : if (RetVT.SimpleTy != MVT::v2f64)
1877 : return 0;
1878 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1879 0 : return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1880 : }
1881 : return 0;
1882 : }
1883 :
1884 0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1885 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1886 0 : return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1887 : }
1888 : return 0;
1889 : }
1890 :
1891 0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
1892 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1893 0 : return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1894 : }
1895 : return 0;
1896 : }
1897 :
1898 0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1899 0 : switch (RetVT.SimpleTy) {
1900 0 : case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
1901 0 : case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
1902 : default: return 0;
1903 : }
1904 : }
1905 :
1906 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
1907 0 : if ((Subtarget->hasDQI())) {
1908 0 : return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
1909 : }
1910 : return 0;
1911 : }
1912 :
1913 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
1914 0 : if ((Subtarget->hasDQI())) {
1915 0 : return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1916 : }
1917 : return 0;
1918 : }
1919 :
1920 0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1921 0 : switch (RetVT.SimpleTy) {
1922 0 : case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
1923 0 : case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
1924 : default: return 0;
1925 : }
1926 : }
1927 :
1928 1101 : unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1929 1101 : switch (VT.SimpleTy) {
1930 1018 : case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
1931 73 : case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
1932 8 : case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1933 2 : case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
1934 0 : case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
1935 0 : case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
1936 0 : case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
1937 0 : case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
1938 : default: return 0;
1939 : }
1940 : }
1941 :
1942 : // FastEmit functions for ISD::TRUNCATE.
1943 :
1944 : unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1945 3 : if (RetVT.SimpleTy != MVT::i8)
1946 : return 0;
1947 3 : if ((Subtarget->is64Bit())) {
1948 2 : return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit);
1949 : }
1950 : return 0;
1951 : }
1952 :
1953 : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
1954 2335 : if ((Subtarget->is64Bit())) {
1955 4662 : return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
1956 : }
1957 : return 0;
1958 : }
1959 :
1960 : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
1961 1628 : return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
1962 : }
1963 :
1964 3963 : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1965 3963 : switch (RetVT.SimpleTy) {
1966 2335 : case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill);
1967 1628 : case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill);
1968 : default: return 0;
1969 : }
1970 : }
1971 :
1972 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
1973 3091 : return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
1974 : }
1975 :
1976 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
1977 18 : return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
1978 : }
1979 :
1980 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1981 5774 : return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit);
1982 : }
1983 :
1984 8883 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1985 8883 : switch (RetVT.SimpleTy) {
1986 3091 : case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill);
1987 18 : case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill);
1988 5774 : case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill);
1989 : default: return 0;
1990 : }
1991 : }
1992 :
1993 6 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1994 6 : if (RetVT.SimpleTy != MVT::v16i8)
1995 : return 0;
1996 6 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1997 4 : return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1998 : }
1999 : return 0;
2000 : }
2001 :
2002 : unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2003 2 : if (RetVT.SimpleTy != MVT::v32i8)
2004 : return 0;
2005 2 : if ((Subtarget->hasBWI())) {
2006 2 : return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2007 : }
2008 : return 0;
2009 : }
2010 :
2011 6 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2012 6 : if (RetVT.SimpleTy != MVT::v8i16)
2013 : return 0;
2014 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2015 4 : return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2016 : }
2017 : return 0;
2018 : }
2019 :
2020 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2021 2 : if ((Subtarget->hasAVX512())) {
2022 2 : return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
2023 : }
2024 : return 0;
2025 : }
2026 :
2027 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
2028 0 : if ((Subtarget->hasAVX512())) {
2029 0 : return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2030 : }
2031 : return 0;
2032 : }
2033 :
2034 2 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2035 2 : switch (RetVT.SimpleTy) {
2036 2 : case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
2037 0 : case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
2038 : default: return 0;
2039 : }
2040 : }
2041 :
2042 4 : unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2043 4 : if (RetVT.SimpleTy != MVT::v4i32)
2044 : return 0;
2045 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2046 2 : return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2047 : }
2048 : return 0;
2049 : }
2050 :
2051 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2052 2 : if ((Subtarget->hasAVX512())) {
2053 2 : return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
2054 : }
2055 : return 0;
2056 : }
2057 :
2058 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2059 2 : if ((Subtarget->hasAVX512())) {
2060 2 : return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2061 : }
2062 : return 0;
2063 : }
2064 :
2065 8 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2066 8 : switch (RetVT.SimpleTy) {
2067 2 : case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
2068 2 : case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
2069 : default: return 0;
2070 : }
2071 : }
2072 :
2073 12880 : unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2074 12880 : switch (VT.SimpleTy) {
2075 3 : case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill);
2076 3963 : case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill);
2077 8883 : case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
2078 6 : case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
2079 2 : case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
2080 6 : case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2081 2 : case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2082 4 : case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2083 8 : case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2084 : default: return 0;
2085 : }
2086 : }
2087 :
2088 : // FastEmit functions for ISD::UINT_TO_FP.
2089 :
2090 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2091 0 : if ((Subtarget->hasVLX())) {
2092 0 : return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2093 : }
2094 : return 0;
2095 : }
2096 :
2097 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2098 0 : if ((Subtarget->hasVLX())) {
2099 0 : return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2100 : }
2101 : return 0;
2102 : }
2103 :
2104 0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2105 0 : switch (RetVT.SimpleTy) {
2106 0 : case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
2107 0 : case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
2108 : default: return 0;
2109 : }
2110 : }
2111 :
2112 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2113 0 : if ((Subtarget->hasVLX())) {
2114 0 : return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2115 : }
2116 : return 0;
2117 : }
2118 :
2119 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2120 0 : if ((Subtarget->hasAVX512())) {
2121 0 : return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2122 : }
2123 : return 0;
2124 : }
2125 :
2126 0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2127 0 : switch (RetVT.SimpleTy) {
2128 0 : case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
2129 0 : case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
2130 : default: return 0;
2131 : }
2132 : }
2133 :
2134 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2135 0 : if (RetVT.SimpleTy != MVT::v16f32)
2136 : return 0;
2137 0 : if ((Subtarget->hasAVX512())) {
2138 0 : return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2139 : }
2140 : return 0;
2141 : }
2142 :
2143 0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2144 0 : if (RetVT.SimpleTy != MVT::v2f64)
2145 : return 0;
2146 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2147 0 : return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2148 : }
2149 : return 0;
2150 : }
2151 :
2152 0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2153 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2154 0 : return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2155 : }
2156 : return 0;
2157 : }
2158 :
2159 0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2160 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2161 0 : return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2162 : }
2163 : return 0;
2164 : }
2165 :
2166 0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2167 0 : switch (RetVT.SimpleTy) {
2168 0 : case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
2169 0 : case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
2170 : default: return 0;
2171 : }
2172 : }
2173 :
2174 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2175 0 : if ((Subtarget->hasDQI())) {
2176 0 : return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2177 : }
2178 : return 0;
2179 : }
2180 :
2181 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2182 0 : if ((Subtarget->hasDQI())) {
2183 0 : return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2184 : }
2185 : return 0;
2186 : }
2187 :
2188 0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2189 0 : switch (RetVT.SimpleTy) {
2190 0 : case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
2191 0 : case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
2192 : default: return 0;
2193 : }
2194 : }
2195 :
2196 0 : unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2197 0 : switch (VT.SimpleTy) {
2198 0 : case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2199 0 : case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2200 0 : case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2201 0 : case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2202 0 : case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2203 0 : case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2204 : default: return 0;
2205 : }
2206 : }
2207 :
2208 : // FastEmit functions for ISD::ZERO_EXTEND.
2209 :
2210 : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2211 87462 : if (RetVT.SimpleTy != MVT::i32)
2212 : return 0;
2213 49928 : return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
2214 : }
2215 :
2216 : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2217 5905 : if (RetVT.SimpleTy != MVT::i32)
2218 : return 0;
2219 5835 : return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
2220 : }
2221 :
2222 128162 : unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2223 128162 : switch (VT.SimpleTy) {
2224 87462 : case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
2225 5905 : case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
2226 : default: return 0;
2227 : }
2228 : }
2229 :
2230 : // FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
2231 :
2232 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2233 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2234 0 : return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
2235 : }
2236 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
2237 0 : return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
2238 : }
2239 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
2240 0 : return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2241 : }
2242 : return 0;
2243 : }
2244 :
2245 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2246 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2247 0 : return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2248 : }
2249 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2250 0 : return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2251 : }
2252 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2253 0 : return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2254 : }
2255 : return 0;
2256 : }
2257 :
2258 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2259 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2260 0 : return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2261 : }
2262 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2263 0 : return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2264 : }
2265 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2266 0 : return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2267 : }
2268 : return 0;
2269 : }
2270 :
2271 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2272 0 : switch (RetVT.SimpleTy) {
2273 0 : case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
2274 0 : case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
2275 0 : case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
2276 : default: return 0;
2277 : }
2278 : }
2279 :
2280 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2281 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2282 0 : return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2283 : }
2284 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2285 0 : return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2286 : }
2287 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2288 0 : return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2289 : }
2290 : return 0;
2291 : }
2292 :
2293 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2294 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2295 0 : return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2296 : }
2297 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2298 0 : return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2299 : }
2300 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2301 0 : return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2302 : }
2303 : return 0;
2304 : }
2305 :
2306 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2307 0 : switch (RetVT.SimpleTy) {
2308 0 : case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
2309 0 : case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
2310 : default: return 0;
2311 : }
2312 : }
2313 :
2314 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2315 0 : if (RetVT.SimpleTy != MVT::v2i64)
2316 : return 0;
2317 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2318 0 : return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2319 : }
2320 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2321 0 : return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2322 : }
2323 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2324 0 : return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2325 : }
2326 : return 0;
2327 : }
2328 :
2329 0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2330 0 : switch (VT.SimpleTy) {
2331 0 : case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2332 0 : case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2333 0 : case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2334 : default: return 0;
2335 : }
2336 : }
2337 :
2338 : // FastEmit functions for X86ISD::CALL.
2339 :
2340 : unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2341 0 : if (RetVT.SimpleTy != MVT::isVoid)
2342 : return 0;
2343 0 : if ((!Subtarget->is64Bit())) {
2344 0 : return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill);
2345 : }
2346 : return 0;
2347 : }
2348 :
2349 0 : unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2350 0 : if (RetVT.SimpleTy != MVT::isVoid)
2351 : return 0;
2352 0 : if ((!Subtarget->is64Bit()) && (Subtarget->useRetpolineIndirectCalls())) {
2353 0 : return fastEmitInst_r(X86::RETPOLINE_CALL32, &X86::GR32RegClass, Op0, Op0IsKill);
2354 : }
2355 0 : if ((!Subtarget->is64Bit()) && (!Subtarget->useRetpolineIndirectCalls())) {
2356 0 : return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill);
2357 : }
2358 : return 0;
2359 : }
2360 :
2361 0 : unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2362 0 : if (RetVT.SimpleTy != MVT::isVoid)
2363 : return 0;
2364 0 : if ((Subtarget->is64Bit()) && (Subtarget->useRetpolineIndirectCalls())) {
2365 0 : return fastEmitInst_r(X86::RETPOLINE_CALL64, &X86::GR64RegClass, Op0, Op0IsKill);
2366 : }
2367 0 : if ((Subtarget->is64Bit()) && (!Subtarget->useRetpolineIndirectCalls())) {
2368 0 : return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill);
2369 : }
2370 : return 0;
2371 : }
2372 :
2373 0 : unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2374 0 : switch (VT.SimpleTy) {
2375 0 : case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
2376 0 : case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
2377 0 : case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
2378 : default: return 0;
2379 : }
2380 : }
2381 :
2382 : // FastEmit functions for X86ISD::COMPRESS.
2383 :
2384 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2385 0 : if (RetVT.SimpleTy != MVT::v16i8)
2386 : return 0;
2387 0 : if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
2388 0 : return fastEmitInst_r(X86::VPCOMPRESSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2389 : }
2390 : return 0;
2391 : }
2392 :
2393 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2394 0 : if (RetVT.SimpleTy != MVT::v32i8)
2395 : return 0;
2396 0 : if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
2397 0 : return fastEmitInst_r(X86::VPCOMPRESSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2398 : }
2399 : return 0;
2400 : }
2401 :
2402 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2403 0 : if (RetVT.SimpleTy != MVT::v64i8)
2404 : return 0;
2405 0 : if ((Subtarget->hasVBMI2())) {
2406 0 : return fastEmitInst_r(X86::VPCOMPRESSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2407 : }
2408 : return 0;
2409 : }
2410 :
2411 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2412 0 : if (RetVT.SimpleTy != MVT::v8i16)
2413 : return 0;
2414 0 : if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
2415 0 : return fastEmitInst_r(X86::VPCOMPRESSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2416 : }
2417 : return 0;
2418 : }
2419 :
2420 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2421 0 : if (RetVT.SimpleTy != MVT::v16i16)
2422 : return 0;
2423 0 : if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
2424 0 : return fastEmitInst_r(X86::VPCOMPRESSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2425 : }
2426 : return 0;
2427 : }
2428 :
2429 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2430 0 : if (RetVT.SimpleTy != MVT::v32i16)
2431 : return 0;
2432 0 : if ((Subtarget->hasVBMI2())) {
2433 0 : return fastEmitInst_r(X86::VPCOMPRESSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2434 : }
2435 : return 0;
2436 : }
2437 :
2438 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2439 0 : if (RetVT.SimpleTy != MVT::v4i32)
2440 : return 0;
2441 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2442 0 : return fastEmitInst_r(X86::VPCOMPRESSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2443 : }
2444 : return 0;
2445 : }
2446 :
2447 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2448 0 : if (RetVT.SimpleTy != MVT::v8i32)
2449 : return 0;
2450 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2451 0 : return fastEmitInst_r(X86::VPCOMPRESSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2452 : }
2453 : return 0;
2454 : }
2455 :
2456 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2457 0 : if (RetVT.SimpleTy != MVT::v16i32)
2458 : return 0;
2459 0 : if ((Subtarget->hasAVX512())) {
2460 0 : return fastEmitInst_r(X86::VPCOMPRESSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2461 : }
2462 : return 0;
2463 : }
2464 :
2465 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2466 0 : if (RetVT.SimpleTy != MVT::v2i64)
2467 : return 0;
2468 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2469 0 : return fastEmitInst_r(X86::VPCOMPRESSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2470 : }
2471 : return 0;
2472 : }
2473 :
2474 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2475 0 : if (RetVT.SimpleTy != MVT::v4i64)
2476 : return 0;
2477 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2478 0 : return fastEmitInst_r(X86::VPCOMPRESSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2479 : }
2480 : return 0;
2481 : }
2482 :
2483 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2484 0 : if (RetVT.SimpleTy != MVT::v8i64)
2485 : return 0;
2486 0 : if ((Subtarget->hasAVX512())) {
2487 0 : return fastEmitInst_r(X86::VPCOMPRESSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2488 : }
2489 : return 0;
2490 : }
2491 :
2492 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2493 0 : if (RetVT.SimpleTy != MVT::v4f32)
2494 : return 0;
2495 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2496 0 : return fastEmitInst_r(X86::VCOMPRESSPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2497 : }
2498 : return 0;
2499 : }
2500 :
2501 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2502 0 : if (RetVT.SimpleTy != MVT::v8f32)
2503 : return 0;
2504 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2505 0 : return fastEmitInst_r(X86::VCOMPRESSPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2506 : }
2507 : return 0;
2508 : }
2509 :
2510 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2511 0 : if (RetVT.SimpleTy != MVT::v16f32)
2512 : return 0;
2513 0 : if ((Subtarget->hasAVX512())) {
2514 0 : return fastEmitInst_r(X86::VCOMPRESSPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2515 : }
2516 : return 0;
2517 : }
2518 :
2519 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2520 0 : if (RetVT.SimpleTy != MVT::v2f64)
2521 : return 0;
2522 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2523 0 : return fastEmitInst_r(X86::VCOMPRESSPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2524 : }
2525 : return 0;
2526 : }
2527 :
2528 0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2529 0 : if (RetVT.SimpleTy != MVT::v4f64)
2530 : return 0;
2531 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2532 0 : return fastEmitInst_r(X86::VCOMPRESSPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2533 : }
2534 : return 0;
2535 : }
2536 :
2537 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2538 0 : if (RetVT.SimpleTy != MVT::v8f64)
2539 : return 0;
2540 0 : if ((Subtarget->hasAVX512())) {
2541 0 : return fastEmitInst_r(X86::VCOMPRESSPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2542 : }
2543 : return 0;
2544 : }
2545 :
2546 0 : unsigned fastEmit_X86ISD_COMPRESS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2547 0 : switch (VT.SimpleTy) {
2548 0 : case MVT::v16i8: return fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2549 0 : case MVT::v32i8: return fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
2550 0 : case MVT::v64i8: return fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
2551 0 : case MVT::v8i16: return fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2552 0 : case MVT::v16i16: return fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
2553 0 : case MVT::v32i16: return fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
2554 0 : case MVT::v4i32: return fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2555 0 : case MVT::v8i32: return fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2556 0 : case MVT::v16i32: return fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2557 0 : case MVT::v2i64: return fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2558 0 : case MVT::v4i64: return fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2559 0 : case MVT::v8i64: return fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2560 0 : case MVT::v4f32: return fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2561 0 : case MVT::v8f32: return fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
2562 0 : case MVT::v16f32: return fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
2563 0 : case MVT::v2f64: return fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
2564 0 : case MVT::v4f64: return fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
2565 0 : case MVT::v8f64: return fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
2566 : default: return 0;
2567 : }
2568 : }
2569 :
2570 : // FastEmit functions for X86ISD::CONFLICT.
2571 :
2572 0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2573 0 : if (RetVT.SimpleTy != MVT::v4i32)
2574 : return 0;
2575 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
2576 0 : return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2577 : }
2578 : return 0;
2579 : }
2580 :
2581 0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2582 0 : if (RetVT.SimpleTy != MVT::v8i32)
2583 : return 0;
2584 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
2585 0 : return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2586 : }
2587 : return 0;
2588 : }
2589 :
2590 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2591 0 : if (RetVT.SimpleTy != MVT::v16i32)
2592 : return 0;
2593 0 : if ((Subtarget->hasCDI())) {
2594 0 : return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2595 : }
2596 : return 0;
2597 : }
2598 :
2599 0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2600 0 : if (RetVT.SimpleTy != MVT::v2i64)
2601 : return 0;
2602 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
2603 0 : return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2604 : }
2605 : return 0;
2606 : }
2607 :
2608 0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2609 0 : if (RetVT.SimpleTy != MVT::v4i64)
2610 : return 0;
2611 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
2612 0 : return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2613 : }
2614 : return 0;
2615 : }
2616 :
2617 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2618 0 : if (RetVT.SimpleTy != MVT::v8i64)
2619 : return 0;
2620 0 : if ((Subtarget->hasCDI())) {
2621 0 : return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2622 : }
2623 : return 0;
2624 : }
2625 :
2626 0 : unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2627 0 : switch (VT.SimpleTy) {
2628 0 : case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2629 0 : case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2630 0 : case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2631 0 : case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2632 0 : case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2633 0 : case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2634 : default: return 0;
2635 : }
2636 : }
2637 :
2638 : // FastEmit functions for X86ISD::CVTP2SI.
2639 :
2640 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2641 0 : if ((Subtarget->hasVLX())) {
2642 0 : return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2643 : }
2644 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2645 0 : return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2646 : }
2647 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2648 0 : return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2649 : }
2650 : return 0;
2651 : }
2652 :
2653 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2654 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2655 0 : return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2656 : }
2657 : return 0;
2658 : }
2659 :
2660 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2661 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2662 0 : return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2663 : }
2664 : return 0;
2665 : }
2666 :
2667 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2668 0 : switch (RetVT.SimpleTy) {
2669 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
2670 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
2671 0 : case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
2672 : default: return 0;
2673 : }
2674 : }
2675 :
2676 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2677 0 : if ((Subtarget->hasVLX())) {
2678 0 : return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2679 : }
2680 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2681 0 : return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2682 : }
2683 : return 0;
2684 : }
2685 :
2686 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2687 0 : if ((Subtarget->hasDQI())) {
2688 0 : return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2689 : }
2690 : return 0;
2691 : }
2692 :
2693 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2694 0 : switch (RetVT.SimpleTy) {
2695 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
2696 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
2697 : default: return 0;
2698 : }
2699 : }
2700 :
2701 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2702 0 : if (RetVT.SimpleTy != MVT::v16i32)
2703 : return 0;
2704 0 : if ((Subtarget->hasAVX512())) {
2705 0 : return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2706 : }
2707 : return 0;
2708 : }
2709 :
2710 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2711 0 : if ((Subtarget->hasVLX())) {
2712 0 : return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2713 : }
2714 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2715 0 : return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2716 : }
2717 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2718 0 : return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2719 : }
2720 : return 0;
2721 : }
2722 :
2723 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2724 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2725 0 : return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2726 : }
2727 : return 0;
2728 : }
2729 :
2730 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2731 0 : switch (RetVT.SimpleTy) {
2732 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
2733 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
2734 : default: return 0;
2735 : }
2736 : }
2737 :
2738 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2739 0 : if ((Subtarget->hasVLX())) {
2740 0 : return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2741 : }
2742 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2743 0 : return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
2744 : }
2745 : return 0;
2746 : }
2747 :
2748 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2749 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2750 0 : return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2751 : }
2752 : return 0;
2753 : }
2754 :
2755 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2756 0 : switch (RetVT.SimpleTy) {
2757 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
2758 0 : case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
2759 : default: return 0;
2760 : }
2761 : }
2762 :
2763 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2764 0 : if ((Subtarget->hasAVX512())) {
2765 0 : return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2766 : }
2767 : return 0;
2768 : }
2769 :
2770 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2771 0 : if ((Subtarget->hasDQI())) {
2772 0 : return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2773 : }
2774 : return 0;
2775 : }
2776 :
2777 0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2778 0 : switch (RetVT.SimpleTy) {
2779 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
2780 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
2781 : default: return 0;
2782 : }
2783 : }
2784 :
2785 0 : unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2786 0 : switch (VT.SimpleTy) {
2787 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2788 0 : case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
2789 0 : case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
2790 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
2791 0 : case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
2792 0 : case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
2793 : default: return 0;
2794 : }
2795 : }
2796 :
2797 : // FastEmit functions for X86ISD::CVTP2UI.
2798 :
2799 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2800 0 : if ((Subtarget->hasVLX())) {
2801 0 : return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2802 : }
2803 : return 0;
2804 : }
2805 :
2806 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2807 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2808 0 : return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2809 : }
2810 : return 0;
2811 : }
2812 :
2813 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2814 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2815 0 : return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2816 : }
2817 : return 0;
2818 : }
2819 :
2820 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2821 0 : switch (RetVT.SimpleTy) {
2822 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
2823 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
2824 0 : case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
2825 : default: return 0;
2826 : }
2827 : }
2828 :
2829 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2830 0 : if ((Subtarget->hasVLX())) {
2831 0 : return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2832 : }
2833 : return 0;
2834 : }
2835 :
2836 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2837 0 : if ((Subtarget->hasDQI())) {
2838 0 : return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2839 : }
2840 : return 0;
2841 : }
2842 :
2843 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2844 0 : switch (RetVT.SimpleTy) {
2845 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
2846 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
2847 : default: return 0;
2848 : }
2849 : }
2850 :
2851 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2852 0 : if (RetVT.SimpleTy != MVT::v16i32)
2853 : return 0;
2854 0 : if ((Subtarget->hasAVX512())) {
2855 0 : return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2856 : }
2857 : return 0;
2858 : }
2859 :
2860 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2861 0 : if ((Subtarget->hasVLX())) {
2862 0 : return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2863 : }
2864 : return 0;
2865 : }
2866 :
2867 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2868 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2869 0 : return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2870 : }
2871 : return 0;
2872 : }
2873 :
2874 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2875 0 : switch (RetVT.SimpleTy) {
2876 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
2877 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
2878 : default: return 0;
2879 : }
2880 : }
2881 :
2882 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2883 0 : if ((Subtarget->hasVLX())) {
2884 0 : return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2885 : }
2886 : return 0;
2887 : }
2888 :
2889 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2890 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2891 0 : return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2892 : }
2893 : return 0;
2894 : }
2895 :
2896 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2897 0 : switch (RetVT.SimpleTy) {
2898 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
2899 0 : case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
2900 : default: return 0;
2901 : }
2902 : }
2903 :
2904 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2905 0 : if ((Subtarget->hasAVX512())) {
2906 0 : return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2907 : }
2908 : return 0;
2909 : }
2910 :
2911 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2912 0 : if ((Subtarget->hasDQI())) {
2913 0 : return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2914 : }
2915 : return 0;
2916 : }
2917 :
2918 0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2919 0 : switch (RetVT.SimpleTy) {
2920 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
2921 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
2922 : default: return 0;
2923 : }
2924 : }
2925 :
2926 0 : unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2927 0 : switch (VT.SimpleTy) {
2928 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2929 0 : case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
2930 0 : case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
2931 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
2932 0 : case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
2933 0 : case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
2934 : default: return 0;
2935 : }
2936 : }
2937 :
2938 : // FastEmit functions for X86ISD::CVTPH2PS.
2939 :
2940 0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2941 0 : if ((Subtarget->hasVLX())) {
2942 0 : return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2943 : }
2944 0 : if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
2945 0 : return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
2946 : }
2947 : return 0;
2948 : }
2949 :
2950 0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2951 0 : if ((Subtarget->hasVLX())) {
2952 0 : return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2953 : }
2954 0 : if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
2955 0 : return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2956 : }
2957 : return 0;
2958 : }
2959 :
2960 0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2961 0 : switch (RetVT.SimpleTy) {
2962 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
2963 0 : case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill);
2964 : default: return 0;
2965 : }
2966 : }
2967 :
2968 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2969 0 : if (RetVT.SimpleTy != MVT::v16f32)
2970 : return 0;
2971 0 : if ((Subtarget->hasAVX512())) {
2972 0 : return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2973 : }
2974 : return 0;
2975 : }
2976 :
2977 0 : unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2978 0 : switch (VT.SimpleTy) {
2979 0 : case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2980 0 : case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
2981 : default: return 0;
2982 : }
2983 : }
2984 :
2985 : // FastEmit functions for X86ISD::CVTS2SI.
2986 :
2987 0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
2988 0 : if ((Subtarget->hasAVX512())) {
2989 0 : return fastEmitInst_r(X86::VCVTSS2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
2990 : }
2991 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2992 0 : return fastEmitInst_r(X86::CVTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
2993 : }
2994 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
2995 0 : return fastEmitInst_r(X86::VCVTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
2996 : }
2997 : return 0;
2998 : }
2999 :
3000 0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3001 0 : if ((Subtarget->hasAVX512())) {
3002 0 : return fastEmitInst_r(X86::VCVTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3003 : }
3004 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3005 0 : return fastEmitInst_r(X86::CVTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3006 : }
3007 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3008 0 : return fastEmitInst_r(X86::VCVTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3009 : }
3010 : return 0;
3011 : }
3012 :
3013 0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3014 0 : switch (RetVT.SimpleTy) {
3015 0 : case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
3016 0 : case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
3017 : default: return 0;
3018 : }
3019 : }
3020 :
3021 0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3022 0 : if ((Subtarget->hasAVX512())) {
3023 0 : return fastEmitInst_r(X86::VCVTSD2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3024 : }
3025 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3026 0 : return fastEmitInst_r(X86::CVTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3027 : }
3028 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3029 0 : return fastEmitInst_r(X86::VCVTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3030 : }
3031 : return 0;
3032 : }
3033 :
3034 0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3035 0 : if ((Subtarget->hasAVX512())) {
3036 0 : return fastEmitInst_r(X86::VCVTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3037 : }
3038 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3039 0 : return fastEmitInst_r(X86::CVTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3040 : }
3041 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3042 0 : return fastEmitInst_r(X86::VCVTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3043 : }
3044 : return 0;
3045 : }
3046 :
3047 0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3048 0 : switch (RetVT.SimpleTy) {
3049 0 : case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
3050 0 : case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
3051 : default: return 0;
3052 : }
3053 : }
3054 :
3055 0 : unsigned fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3056 0 : switch (VT.SimpleTy) {
3057 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3058 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3059 : default: return 0;
3060 : }
3061 : }
3062 :
3063 : // FastEmit functions for X86ISD::CVTS2UI.
3064 :
3065 : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3066 0 : if ((Subtarget->hasAVX512())) {
3067 0 : return fastEmitInst_r(X86::VCVTSS2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3068 : }
3069 : return 0;
3070 : }
3071 :
3072 : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3073 0 : if ((Subtarget->hasAVX512())) {
3074 0 : return fastEmitInst_r(X86::VCVTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3075 : }
3076 : return 0;
3077 : }
3078 :
3079 0 : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3080 0 : switch (RetVT.SimpleTy) {
3081 0 : case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
3082 0 : case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
3083 : default: return 0;
3084 : }
3085 : }
3086 :
3087 : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3088 0 : if ((Subtarget->hasAVX512())) {
3089 0 : return fastEmitInst_r(X86::VCVTSD2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3090 : }
3091 : return 0;
3092 : }
3093 :
3094 : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3095 0 : if ((Subtarget->hasAVX512())) {
3096 0 : return fastEmitInst_r(X86::VCVTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3097 : }
3098 : return 0;
3099 : }
3100 :
3101 0 : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3102 0 : switch (RetVT.SimpleTy) {
3103 0 : case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
3104 0 : case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
3105 : default: return 0;
3106 : }
3107 : }
3108 :
3109 0 : unsigned fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3110 0 : switch (VT.SimpleTy) {
3111 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3112 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3113 : default: return 0;
3114 : }
3115 : }
3116 :
3117 : // FastEmit functions for X86ISD::CVTSI2P.
3118 :
3119 0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3120 0 : if (RetVT.SimpleTy != MVT::v2f64)
3121 : return 0;
3122 0 : if ((Subtarget->hasVLX())) {
3123 0 : return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3124 : }
3125 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3126 0 : return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
3127 : }
3128 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3129 0 : return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
3130 : }
3131 : return 0;
3132 : }
3133 :
3134 0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3135 0 : if (RetVT.SimpleTy != MVT::v4f32)
3136 : return 0;
3137 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3138 0 : return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3139 : }
3140 : return 0;
3141 : }
3142 :
3143 0 : unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3144 0 : switch (VT.SimpleTy) {
3145 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3146 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3147 : default: return 0;
3148 : }
3149 : }
3150 :
3151 : // FastEmit functions for X86ISD::CVTTP2SI.
3152 :
3153 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3154 0 : if ((Subtarget->hasVLX())) {
3155 0 : return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3156 : }
3157 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3158 0 : return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3159 : }
3160 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3161 0 : return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3162 : }
3163 : return 0;
3164 : }
3165 :
3166 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3167 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3168 0 : return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3169 : }
3170 : return 0;
3171 : }
3172 :
3173 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3174 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3175 0 : return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3176 : }
3177 : return 0;
3178 : }
3179 :
3180 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3181 0 : switch (RetVT.SimpleTy) {
3182 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
3183 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
3184 0 : case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
3185 : default: return 0;
3186 : }
3187 : }
3188 :
3189 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3190 0 : if ((Subtarget->hasVLX())) {
3191 0 : return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3192 : }
3193 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3194 0 : return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3195 : }
3196 : return 0;
3197 : }
3198 :
3199 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3200 0 : if ((Subtarget->hasDQI())) {
3201 0 : return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3202 : }
3203 : return 0;
3204 : }
3205 :
3206 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3207 0 : switch (RetVT.SimpleTy) {
3208 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
3209 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
3210 : default: return 0;
3211 : }
3212 : }
3213 :
3214 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3215 0 : if (RetVT.SimpleTy != MVT::v16i32)
3216 : return 0;
3217 0 : if ((Subtarget->hasAVX512())) {
3218 0 : return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3219 : }
3220 : return 0;
3221 : }
3222 :
3223 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3224 0 : if ((Subtarget->hasVLX())) {
3225 0 : return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3226 : }
3227 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3228 0 : return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3229 : }
3230 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3231 0 : return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3232 : }
3233 : return 0;
3234 : }
3235 :
3236 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3237 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3238 0 : return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3239 : }
3240 : return 0;
3241 : }
3242 :
3243 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3244 0 : switch (RetVT.SimpleTy) {
3245 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
3246 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
3247 : default: return 0;
3248 : }
3249 : }
3250 :
3251 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3252 0 : if ((Subtarget->hasVLX())) {
3253 0 : return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3254 : }
3255 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3256 0 : return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
3257 : }
3258 : return 0;
3259 : }
3260 :
3261 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3262 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3263 0 : return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3264 : }
3265 : return 0;
3266 : }
3267 :
3268 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3269 0 : switch (RetVT.SimpleTy) {
3270 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
3271 0 : case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
3272 : default: return 0;
3273 : }
3274 : }
3275 :
3276 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3277 0 : if ((Subtarget->hasAVX512())) {
3278 0 : return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
3279 : }
3280 : return 0;
3281 : }
3282 :
3283 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3284 0 : if ((Subtarget->hasDQI())) {
3285 0 : return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3286 : }
3287 : return 0;
3288 : }
3289 :
3290 0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3291 0 : switch (RetVT.SimpleTy) {
3292 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
3293 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
3294 : default: return 0;
3295 : }
3296 : }
3297 :
3298 0 : unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3299 0 : switch (VT.SimpleTy) {
3300 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3301 0 : case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3302 0 : case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3303 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3304 0 : case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3305 0 : case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3306 : default: return 0;
3307 : }
3308 : }
3309 :
3310 : // FastEmit functions for X86ISD::CVTTP2UI.
3311 :
3312 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3313 0 : if ((Subtarget->hasVLX())) {
3314 0 : return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3315 : }
3316 : return 0;
3317 : }
3318 :
3319 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3320 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3321 0 : return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3322 : }
3323 : return 0;
3324 : }
3325 :
3326 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3327 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3328 0 : return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3329 : }
3330 : return 0;
3331 : }
3332 :
3333 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3334 0 : switch (RetVT.SimpleTy) {
3335 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
3336 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
3337 0 : case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
3338 : default: return 0;
3339 : }
3340 : }
3341 :
3342 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3343 0 : if ((Subtarget->hasVLX())) {
3344 0 : return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3345 : }
3346 : return 0;
3347 : }
3348 :
3349 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3350 0 : if ((Subtarget->hasDQI())) {
3351 0 : return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3352 : }
3353 : return 0;
3354 : }
3355 :
3356 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3357 0 : switch (RetVT.SimpleTy) {
3358 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
3359 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
3360 : default: return 0;
3361 : }
3362 : }
3363 :
3364 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3365 0 : if (RetVT.SimpleTy != MVT::v16i32)
3366 : return 0;
3367 0 : if ((Subtarget->hasAVX512())) {
3368 0 : return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3369 : }
3370 : return 0;
3371 : }
3372 :
3373 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3374 0 : if ((Subtarget->hasVLX())) {
3375 0 : return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3376 : }
3377 : return 0;
3378 : }
3379 :
3380 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3381 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3382 0 : return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3383 : }
3384 : return 0;
3385 : }
3386 :
3387 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3388 0 : switch (RetVT.SimpleTy) {
3389 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
3390 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
3391 : default: return 0;
3392 : }
3393 : }
3394 :
3395 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3396 0 : if ((Subtarget->hasVLX())) {
3397 0 : return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3398 : }
3399 : return 0;
3400 : }
3401 :
3402 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3403 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3404 0 : return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3405 : }
3406 : return 0;
3407 : }
3408 :
3409 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3410 0 : switch (RetVT.SimpleTy) {
3411 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
3412 0 : case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
3413 : default: return 0;
3414 : }
3415 : }
3416 :
3417 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3418 0 : if ((Subtarget->hasAVX512())) {
3419 0 : return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
3420 : }
3421 : return 0;
3422 : }
3423 :
3424 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3425 0 : if ((Subtarget->hasDQI())) {
3426 0 : return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3427 : }
3428 : return 0;
3429 : }
3430 :
3431 0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3432 0 : switch (RetVT.SimpleTy) {
3433 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
3434 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
3435 : default: return 0;
3436 : }
3437 : }
3438 :
3439 0 : unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3440 0 : switch (VT.SimpleTy) {
3441 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3442 0 : case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3443 0 : case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3444 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3445 0 : case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3446 0 : case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3447 : default: return 0;
3448 : }
3449 : }
3450 :
3451 : // FastEmit functions for X86ISD::CVTTS2SI.
3452 :
3453 0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3454 0 : if ((Subtarget->hasAVX512())) {
3455 0 : return fastEmitInst_r(X86::VCVTTSS2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3456 : }
3457 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3458 0 : return fastEmitInst_r(X86::CVTTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3459 : }
3460 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3461 0 : return fastEmitInst_r(X86::VCVTTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3462 : }
3463 : return 0;
3464 : }
3465 :
3466 0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3467 0 : if ((Subtarget->hasAVX512())) {
3468 0 : return fastEmitInst_r(X86::VCVTTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3469 : }
3470 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3471 0 : return fastEmitInst_r(X86::CVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3472 : }
3473 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3474 0 : return fastEmitInst_r(X86::VCVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3475 : }
3476 : return 0;
3477 : }
3478 :
3479 0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3480 0 : switch (RetVT.SimpleTy) {
3481 0 : case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
3482 0 : case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
3483 : default: return 0;
3484 : }
3485 : }
3486 :
3487 0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3488 0 : if ((Subtarget->hasAVX512())) {
3489 0 : return fastEmitInst_r(X86::VCVTTSD2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3490 : }
3491 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3492 0 : return fastEmitInst_r(X86::CVTTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3493 : }
3494 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3495 0 : return fastEmitInst_r(X86::VCVTTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3496 : }
3497 : return 0;
3498 : }
3499 :
3500 0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3501 0 : if ((Subtarget->hasAVX512())) {
3502 0 : return fastEmitInst_r(X86::VCVTTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3503 : }
3504 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3505 0 : return fastEmitInst_r(X86::CVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3506 : }
3507 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3508 0 : return fastEmitInst_r(X86::VCVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3509 : }
3510 : return 0;
3511 : }
3512 :
3513 0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3514 0 : switch (RetVT.SimpleTy) {
3515 0 : case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
3516 0 : case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
3517 : default: return 0;
3518 : }
3519 : }
3520 :
3521 0 : unsigned fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3522 0 : switch (VT.SimpleTy) {
3523 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3524 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3525 : default: return 0;
3526 : }
3527 : }
3528 :
3529 : // FastEmit functions for X86ISD::CVTTS2UI.
3530 :
3531 : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3532 0 : if ((Subtarget->hasAVX512())) {
3533 0 : return fastEmitInst_r(X86::VCVTTSS2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3534 : }
3535 : return 0;
3536 : }
3537 :
3538 : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3539 0 : if ((Subtarget->hasAVX512())) {
3540 0 : return fastEmitInst_r(X86::VCVTTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3541 : }
3542 : return 0;
3543 : }
3544 :
3545 0 : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3546 0 : switch (RetVT.SimpleTy) {
3547 0 : case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
3548 0 : case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
3549 : default: return 0;
3550 : }
3551 : }
3552 :
3553 : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3554 0 : if ((Subtarget->hasAVX512())) {
3555 0 : return fastEmitInst_r(X86::VCVTTSD2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3556 : }
3557 : return 0;
3558 : }
3559 :
3560 : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3561 0 : if ((Subtarget->hasAVX512())) {
3562 0 : return fastEmitInst_r(X86::VCVTTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3563 : }
3564 : return 0;
3565 : }
3566 :
3567 0 : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3568 0 : switch (RetVT.SimpleTy) {
3569 0 : case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
3570 0 : case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
3571 : default: return 0;
3572 : }
3573 : }
3574 :
3575 0 : unsigned fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3576 0 : switch (VT.SimpleTy) {
3577 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3578 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3579 : default: return 0;
3580 : }
3581 : }
3582 :
3583 : // FastEmit functions for X86ISD::CVTUI2P.
3584 :
3585 : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3586 0 : if (RetVT.SimpleTy != MVT::v2f64)
3587 : return 0;
3588 0 : if ((Subtarget->hasVLX())) {
3589 0 : return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3590 : }
3591 : return 0;
3592 : }
3593 :
3594 0 : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3595 0 : if (RetVT.SimpleTy != MVT::v4f32)
3596 : return 0;
3597 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3598 0 : return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3599 : }
3600 : return 0;
3601 : }
3602 :
3603 0 : unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3604 0 : switch (VT.SimpleTy) {
3605 0 : case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3606 0 : case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3607 : default: return 0;
3608 : }
3609 : }
3610 :
3611 : // FastEmit functions for X86ISD::EH_RETURN.
3612 :
3613 : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3614 0 : if (RetVT.SimpleTy != MVT::isVoid)
3615 : return 0;
3616 0 : return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill);
3617 : }
3618 :
3619 : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3620 0 : if (RetVT.SimpleTy != MVT::isVoid)
3621 : return 0;
3622 0 : return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill);
3623 : }
3624 :
3625 0 : unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3626 0 : switch (VT.SimpleTy) {
3627 0 : case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
3628 0 : case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill);
3629 : default: return 0;
3630 : }
3631 : }
3632 :
3633 : // FastEmit functions for X86ISD::EXPAND.
3634 :
3635 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3636 0 : if (RetVT.SimpleTy != MVT::v16i8)
3637 : return 0;
3638 0 : if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
3639 0 : return fastEmitInst_r(X86::VPEXPANDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3640 : }
3641 : return 0;
3642 : }
3643 :
3644 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3645 0 : if (RetVT.SimpleTy != MVT::v32i8)
3646 : return 0;
3647 0 : if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
3648 0 : return fastEmitInst_r(X86::VPEXPANDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3649 : }
3650 : return 0;
3651 : }
3652 :
3653 : unsigned fastEmit_X86ISD_EXPAND_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3654 0 : if (RetVT.SimpleTy != MVT::v64i8)
3655 : return 0;
3656 0 : if ((Subtarget->hasVBMI2())) {
3657 0 : return fastEmitInst_r(X86::VPEXPANDBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3658 : }
3659 : return 0;
3660 : }
3661 :
3662 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3663 0 : if (RetVT.SimpleTy != MVT::v8i16)
3664 : return 0;
3665 0 : if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
3666 0 : return fastEmitInst_r(X86::VPEXPANDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3667 : }
3668 : return 0;
3669 : }
3670 :
3671 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3672 0 : if (RetVT.SimpleTy != MVT::v16i16)
3673 : return 0;
3674 0 : if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
3675 0 : return fastEmitInst_r(X86::VPEXPANDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3676 : }
3677 : return 0;
3678 : }
3679 :
3680 : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3681 0 : if (RetVT.SimpleTy != MVT::v32i16)
3682 : return 0;
3683 0 : if ((Subtarget->hasVBMI2())) {
3684 0 : return fastEmitInst_r(X86::VPEXPANDWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3685 : }
3686 : return 0;
3687 : }
3688 :
3689 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3690 0 : if (RetVT.SimpleTy != MVT::v4i32)
3691 : return 0;
3692 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3693 0 : return fastEmitInst_r(X86::VPEXPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3694 : }
3695 : return 0;
3696 : }
3697 :
3698 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3699 0 : if (RetVT.SimpleTy != MVT::v8i32)
3700 : return 0;
3701 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3702 0 : return fastEmitInst_r(X86::VPEXPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3703 : }
3704 : return 0;
3705 : }
3706 :
3707 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3708 0 : if (RetVT.SimpleTy != MVT::v16i32)
3709 : return 0;
3710 0 : if ((Subtarget->hasAVX512())) {
3711 0 : return fastEmitInst_r(X86::VPEXPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3712 : }
3713 : return 0;
3714 : }
3715 :
3716 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3717 0 : if (RetVT.SimpleTy != MVT::v2i64)
3718 : return 0;
3719 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3720 0 : return fastEmitInst_r(X86::VPEXPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3721 : }
3722 : return 0;
3723 : }
3724 :
3725 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3726 0 : if (RetVT.SimpleTy != MVT::v4i64)
3727 : return 0;
3728 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3729 0 : return fastEmitInst_r(X86::VPEXPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3730 : }
3731 : return 0;
3732 : }
3733 :
3734 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3735 0 : if (RetVT.SimpleTy != MVT::v8i64)
3736 : return 0;
3737 0 : if ((Subtarget->hasAVX512())) {
3738 0 : return fastEmitInst_r(X86::VPEXPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3739 : }
3740 : return 0;
3741 : }
3742 :
3743 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3744 0 : if (RetVT.SimpleTy != MVT::v4f32)
3745 : return 0;
3746 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3747 0 : return fastEmitInst_r(X86::VEXPANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3748 : }
3749 : return 0;
3750 : }
3751 :
3752 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3753 0 : if (RetVT.SimpleTy != MVT::v8f32)
3754 : return 0;
3755 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3756 0 : return fastEmitInst_r(X86::VEXPANDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3757 : }
3758 : return 0;
3759 : }
3760 :
3761 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3762 0 : if (RetVT.SimpleTy != MVT::v16f32)
3763 : return 0;
3764 0 : if ((Subtarget->hasAVX512())) {
3765 0 : return fastEmitInst_r(X86::VEXPANDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3766 : }
3767 : return 0;
3768 : }
3769 :
3770 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3771 0 : if (RetVT.SimpleTy != MVT::v2f64)
3772 : return 0;
3773 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3774 0 : return fastEmitInst_r(X86::VEXPANDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3775 : }
3776 : return 0;
3777 : }
3778 :
3779 0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3780 0 : if (RetVT.SimpleTy != MVT::v4f64)
3781 : return 0;
3782 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3783 0 : return fastEmitInst_r(X86::VEXPANDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3784 : }
3785 : return 0;
3786 : }
3787 :
3788 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3789 0 : if (RetVT.SimpleTy != MVT::v8f64)
3790 : return 0;
3791 0 : if ((Subtarget->hasAVX512())) {
3792 0 : return fastEmitInst_r(X86::VEXPANDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3793 : }
3794 : return 0;
3795 : }
3796 :
3797 0 : unsigned fastEmit_X86ISD_EXPAND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3798 0 : switch (VT.SimpleTy) {
3799 0 : case MVT::v16i8: return fastEmit_X86ISD_EXPAND_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
3800 0 : case MVT::v32i8: return fastEmit_X86ISD_EXPAND_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
3801 0 : case MVT::v64i8: return fastEmit_X86ISD_EXPAND_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
3802 0 : case MVT::v8i16: return fastEmit_X86ISD_EXPAND_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
3803 0 : case MVT::v16i16: return fastEmit_X86ISD_EXPAND_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
3804 0 : case MVT::v32i16: return fastEmit_X86ISD_EXPAND_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
3805 0 : case MVT::v4i32: return fastEmit_X86ISD_EXPAND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3806 0 : case MVT::v8i32: return fastEmit_X86ISD_EXPAND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
3807 0 : case MVT::v16i32: return fastEmit_X86ISD_EXPAND_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
3808 0 : case MVT::v2i64: return fastEmit_X86ISD_EXPAND_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3809 0 : case MVT::v4i64: return fastEmit_X86ISD_EXPAND_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
3810 0 : case MVT::v8i64: return fastEmit_X86ISD_EXPAND_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
3811 0 : case MVT::v4f32: return fastEmit_X86ISD_EXPAND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3812 0 : case MVT::v8f32: return fastEmit_X86ISD_EXPAND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3813 0 : case MVT::v16f32: return fastEmit_X86ISD_EXPAND_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3814 0 : case MVT::v2f64: return fastEmit_X86ISD_EXPAND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3815 0 : case MVT::v4f64: return fastEmit_X86ISD_EXPAND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3816 0 : case MVT::v8f64: return fastEmit_X86ISD_EXPAND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3817 : default: return 0;
3818 : }
3819 : }
3820 :
3821 : // FastEmit functions for X86ISD::FRCP.
3822 :
3823 0 : unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3824 0 : if (RetVT.SimpleTy != MVT::f32)
3825 : return 0;
3826 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3827 0 : return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill);
3828 : }
3829 : return 0;
3830 : }
3831 :
3832 0 : unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3833 0 : if (RetVT.SimpleTy != MVT::v4f32)
3834 : return 0;
3835 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3836 0 : return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
3837 : }
3838 0 : if ((Subtarget->hasAVX())) {
3839 0 : return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
3840 : }
3841 : return 0;
3842 : }
3843 :
3844 : unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3845 0 : if (RetVT.SimpleTy != MVT::v8f32)
3846 : return 0;
3847 0 : if ((Subtarget->hasAVX())) {
3848 0 : return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
3849 : }
3850 : return 0;
3851 : }
3852 :
3853 0 : unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3854 0 : switch (VT.SimpleTy) {
3855 0 : case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill);
3856 0 : case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3857 0 : case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3858 : default: return 0;
3859 : }
3860 : }
3861 :
3862 : // FastEmit functions for X86ISD::FRSQRT.
3863 :
3864 0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3865 0 : if (RetVT.SimpleTy != MVT::f32)
3866 : return 0;
3867 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3868 0 : return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
3869 : }
3870 : return 0;
3871 : }
3872 :
3873 0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3874 0 : if (RetVT.SimpleTy != MVT::v4f32)
3875 : return 0;
3876 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3877 0 : return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
3878 : }
3879 0 : if ((Subtarget->hasAVX())) {
3880 0 : return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
3881 : }
3882 : return 0;
3883 : }
3884 :
3885 : unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3886 0 : if (RetVT.SimpleTy != MVT::v8f32)
3887 : return 0;
3888 0 : if ((Subtarget->hasAVX())) {
3889 0 : return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
3890 : }
3891 : return 0;
3892 : }
3893 :
3894 0 : unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3895 0 : switch (VT.SimpleTy) {
3896 0 : case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
3897 0 : case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3898 0 : case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3899 : default: return 0;
3900 : }
3901 : }
3902 :
3903 : // FastEmit functions for X86ISD::MMX_MOVD2W.
3904 :
3905 : unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3906 0 : if (RetVT.SimpleTy != MVT::i32)
3907 : return 0;
3908 0 : if ((Subtarget->hasMMX())) {
3909 0 : return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill);
3910 : }
3911 : return 0;
3912 : }
3913 :
3914 0 : unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3915 0 : switch (VT.SimpleTy) {
3916 0 : case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
3917 : default: return 0;
3918 : }
3919 : }
3920 :
3921 : // FastEmit functions for X86ISD::MMX_MOVW2D.
3922 :
3923 : unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3924 0 : if (RetVT.SimpleTy != MVT::x86mmx)
3925 : return 0;
3926 0 : if ((Subtarget->hasMMX())) {
3927 0 : return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
3928 : }
3929 : return 0;
3930 : }
3931 :
3932 0 : unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3933 0 : switch (VT.SimpleTy) {
3934 0 : case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill);
3935 : default: return 0;
3936 : }
3937 : }
3938 :
3939 : // FastEmit functions for X86ISD::MOVDDUP.
3940 :
3941 0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3942 0 : if (RetVT.SimpleTy != MVT::v2f64)
3943 : return 0;
3944 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
3945 0 : return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3946 : }
3947 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3948 0 : return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3949 : }
3950 : return 0;
3951 : }
3952 :
3953 0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3954 0 : if (RetVT.SimpleTy != MVT::v4f64)
3955 : return 0;
3956 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3957 0 : return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3958 : }
3959 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3960 0 : return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3961 : }
3962 : return 0;
3963 : }
3964 :
3965 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3966 0 : if (RetVT.SimpleTy != MVT::v8f64)
3967 : return 0;
3968 0 : if ((Subtarget->hasAVX512())) {
3969 0 : return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3970 : }
3971 : return 0;
3972 : }
3973 :
3974 0 : unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3975 0 : switch (VT.SimpleTy) {
3976 0 : case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3977 0 : case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3978 0 : case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3979 : default: return 0;
3980 : }
3981 : }
3982 :
3983 : // FastEmit functions for X86ISD::MOVDQ2Q.
3984 :
3985 : unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3986 0 : if (RetVT.SimpleTy != MVT::x86mmx)
3987 : return 0;
3988 0 : return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
3989 : }
3990 :
3991 : unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3992 0 : switch (VT.SimpleTy) {
3993 0 : case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3994 : default: return 0;
3995 : }
3996 : }
3997 :
3998 : // FastEmit functions for X86ISD::MOVMSK.
3999 :
4000 0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4001 0 : if (RetVT.SimpleTy != MVT::i32)
4002 : return 0;
4003 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4004 0 : return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
4005 : }
4006 0 : if ((Subtarget->hasAVX())) {
4007 0 : return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
4008 : }
4009 : return 0;
4010 : }
4011 :
4012 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4013 0 : if (RetVT.SimpleTy != MVT::i32)
4014 : return 0;
4015 0 : if ((Subtarget->hasAVX2())) {
4016 0 : return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4017 : }
4018 : return 0;
4019 : }
4020 :
4021 0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4022 0 : if (RetVT.SimpleTy != MVT::i32)
4023 : return 0;
4024 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4025 0 : return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
4026 : }
4027 0 : if ((Subtarget->hasAVX())) {
4028 0 : return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
4029 : }
4030 : return 0;
4031 : }
4032 :
4033 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4034 0 : if (RetVT.SimpleTy != MVT::i32)
4035 : return 0;
4036 0 : if ((Subtarget->hasAVX())) {
4037 0 : return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4038 : }
4039 : return 0;
4040 : }
4041 :
4042 0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4043 0 : if (RetVT.SimpleTy != MVT::i32)
4044 : return 0;
4045 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4046 0 : return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
4047 : }
4048 0 : if ((Subtarget->hasAVX())) {
4049 0 : return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
4050 : }
4051 : return 0;
4052 : }
4053 :
4054 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4055 0 : if (RetVT.SimpleTy != MVT::i32)
4056 : return 0;
4057 0 : if ((Subtarget->hasAVX())) {
4058 0 : return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4059 : }
4060 : return 0;
4061 : }
4062 :
4063 0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4064 0 : if (RetVT.SimpleTy != MVT::i32)
4065 : return 0;
4066 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4067 0 : return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
4068 : }
4069 0 : if ((Subtarget->hasAVX())) {
4070 0 : return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
4071 : }
4072 : return 0;
4073 : }
4074 :
4075 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4076 0 : if (RetVT.SimpleTy != MVT::i32)
4077 : return 0;
4078 0 : if ((Subtarget->hasAVX())) {
4079 0 : return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4080 : }
4081 : return 0;
4082 : }
4083 :
4084 0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4085 0 : if (RetVT.SimpleTy != MVT::i32)
4086 : return 0;
4087 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4088 0 : return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
4089 : }
4090 0 : if ((Subtarget->hasAVX())) {
4091 0 : return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
4092 : }
4093 : return 0;
4094 : }
4095 :
4096 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4097 0 : if (RetVT.SimpleTy != MVT::i32)
4098 : return 0;
4099 0 : if ((Subtarget->hasAVX())) {
4100 0 : return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4101 : }
4102 : return 0;
4103 : }
4104 :
4105 0 : unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4106 0 : switch (VT.SimpleTy) {
4107 0 : case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
4108 0 : case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
4109 0 : case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4110 0 : case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
4111 0 : case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4112 0 : case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
4113 0 : case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4114 0 : case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4115 0 : case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4116 0 : case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4117 : default: return 0;
4118 : }
4119 : }
4120 :
4121 : // FastEmit functions for X86ISD::MOVSHDUP.
4122 :
4123 0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4124 0 : if (RetVT.SimpleTy != MVT::v4i32)
4125 : return 0;
4126 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4127 0 : return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4128 : }
4129 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4130 0 : return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4131 : }
4132 : return 0;
4133 : }
4134 :
4135 0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4136 0 : if (RetVT.SimpleTy != MVT::v8i32)
4137 : return 0;
4138 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4139 0 : return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4140 : }
4141 : return 0;
4142 : }
4143 :
4144 0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4145 0 : if (RetVT.SimpleTy != MVT::v4f32)
4146 : return 0;
4147 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4148 0 : return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4149 : }
4150 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4151 0 : return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4152 : }
4153 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4154 0 : return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4155 : }
4156 : return 0;
4157 : }
4158 :
4159 0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4160 0 : if (RetVT.SimpleTy != MVT::v8f32)
4161 : return 0;
4162 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4163 0 : return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4164 : }
4165 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4166 0 : return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4167 : }
4168 : return 0;
4169 : }
4170 :
4171 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4172 0 : if (RetVT.SimpleTy != MVT::v16f32)
4173 : return 0;
4174 0 : if ((Subtarget->hasAVX512())) {
4175 0 : return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4176 : }
4177 : return 0;
4178 : }
4179 :
4180 0 : unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4181 0 : switch (VT.SimpleTy) {
4182 0 : case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4183 0 : case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
4184 0 : case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4185 0 : case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4186 0 : case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4187 : default: return 0;
4188 : }
4189 : }
4190 :
4191 : // FastEmit functions for X86ISD::MOVSLDUP.
4192 :
4193 0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4194 0 : if (RetVT.SimpleTy != MVT::v4i32)
4195 : return 0;
4196 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4197 0 : return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4198 : }
4199 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4200 0 : return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4201 : }
4202 : return 0;
4203 : }
4204 :
4205 0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4206 0 : if (RetVT.SimpleTy != MVT::v8i32)
4207 : return 0;
4208 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4209 0 : return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4210 : }
4211 : return 0;
4212 : }
4213 :
4214 0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4215 0 : if (RetVT.SimpleTy != MVT::v4f32)
4216 : return 0;
4217 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4218 0 : return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4219 : }
4220 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4221 0 : return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4222 : }
4223 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4224 0 : return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4225 : }
4226 : return 0;
4227 : }
4228 :
4229 0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4230 0 : if (RetVT.SimpleTy != MVT::v8f32)
4231 : return 0;
4232 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4233 0 : return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4234 : }
4235 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4236 0 : return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4237 : }
4238 : return 0;
4239 : }
4240 :
4241 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4242 0 : if (RetVT.SimpleTy != MVT::v16f32)
4243 : return 0;
4244 0 : if ((Subtarget->hasAVX512())) {
4245 0 : return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4246 : }
4247 : return 0;
4248 : }
4249 :
4250 0 : unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4251 0 : switch (VT.SimpleTy) {
4252 0 : case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4253 0 : case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
4254 0 : case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4255 0 : case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4256 0 : case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4257 : default: return 0;
4258 : }
4259 : }
4260 :
4261 : // FastEmit functions for X86ISD::NT_BRIND.
4262 :
4263 : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4264 0 : if (RetVT.SimpleTy != MVT::isVoid)
4265 : return 0;
4266 0 : if ((!Subtarget->is64Bit())) {
4267 0 : return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
4268 : }
4269 : return 0;
4270 : }
4271 :
4272 : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4273 0 : if (RetVT.SimpleTy != MVT::isVoid)
4274 : return 0;
4275 0 : if ((!Subtarget->is64Bit())) {
4276 0 : return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
4277 : }
4278 : return 0;
4279 : }
4280 :
4281 : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4282 0 : if (RetVT.SimpleTy != MVT::isVoid)
4283 : return 0;
4284 0 : if ((Subtarget->is64Bit())) {
4285 0 : return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
4286 : }
4287 : return 0;
4288 : }
4289 :
4290 0 : unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4291 0 : switch (VT.SimpleTy) {
4292 0 : case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
4293 0 : case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
4294 0 : case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
4295 : default: return 0;
4296 : }
4297 : }
4298 :
4299 : // FastEmit functions for X86ISD::NT_CALL.
4300 :
4301 : unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4302 0 : if (RetVT.SimpleTy != MVT::isVoid)
4303 : return 0;
4304 0 : if ((!Subtarget->is64Bit())) {
4305 0 : return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
4306 : }
4307 : return 0;
4308 : }
4309 :
4310 : unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4311 0 : if (RetVT.SimpleTy != MVT::isVoid)
4312 : return 0;
4313 0 : if ((!Subtarget->is64Bit())) {
4314 0 : return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
4315 : }
4316 : return 0;
4317 : }
4318 :
4319 : unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4320 0 : if (RetVT.SimpleTy != MVT::isVoid)
4321 : return 0;
4322 0 : if ((Subtarget->is64Bit())) {
4323 0 : return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
4324 : }
4325 : return 0;
4326 : }
4327 :
4328 0 : unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4329 0 : switch (VT.SimpleTy) {
4330 0 : case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
4331 0 : case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
4332 0 : case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
4333 : default: return 0;
4334 : }
4335 : }
4336 :
4337 : // FastEmit functions for X86ISD::PHMINPOS.
4338 :
4339 0 : unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4340 0 : if (RetVT.SimpleTy != MVT::v8i16)
4341 : return 0;
4342 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
4343 0 : return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
4344 : }
4345 0 : if ((Subtarget->hasAVX())) {
4346 0 : return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
4347 : }
4348 : return 0;
4349 : }
4350 :
4351 : unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4352 0 : switch (VT.SimpleTy) {
4353 0 : case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
4354 : default: return 0;
4355 : }
4356 : }
4357 :
4358 : // FastEmit functions for X86ISD::RCP14.
4359 :
4360 : unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4361 0 : if (RetVT.SimpleTy != MVT::v4f32)
4362 : return 0;
4363 0 : if ((Subtarget->hasVLX())) {
4364 0 : return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4365 : }
4366 : return 0;
4367 : }
4368 :
4369 : unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4370 0 : if (RetVT.SimpleTy != MVT::v8f32)
4371 : return 0;
4372 0 : if ((Subtarget->hasVLX())) {
4373 0 : return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4374 : }
4375 : return 0;
4376 : }
4377 :
4378 : unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4379 0 : if (RetVT.SimpleTy != MVT::v16f32)
4380 : return 0;
4381 0 : if ((Subtarget->hasAVX512())) {
4382 0 : return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
4383 : }
4384 : return 0;
4385 : }
4386 :
4387 : unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4388 0 : if (RetVT.SimpleTy != MVT::v2f64)
4389 : return 0;
4390 0 : if ((Subtarget->hasVLX())) {
4391 0 : return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4392 : }
4393 : return 0;
4394 : }
4395 :
4396 : unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4397 0 : if (RetVT.SimpleTy != MVT::v4f64)
4398 : return 0;
4399 0 : if ((Subtarget->hasVLX())) {
4400 0 : return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4401 : }
4402 : return 0;
4403 : }
4404 :
4405 : unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4406 0 : if (RetVT.SimpleTy != MVT::v8f64)
4407 : return 0;
4408 0 : if ((Subtarget->hasAVX512())) {
4409 0 : return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4410 : }
4411 : return 0;
4412 : }
4413 :
4414 0 : unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4415 0 : switch (VT.SimpleTy) {
4416 0 : case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4417 0 : case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4418 0 : case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4419 0 : case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4420 0 : case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4421 0 : case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4422 : default: return 0;
4423 : }
4424 : }
4425 :
4426 : // FastEmit functions for X86ISD::RSQRT14.
4427 :
4428 : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4429 0 : if (RetVT.SimpleTy != MVT::v4f32)
4430 : return 0;
4431 0 : if ((Subtarget->hasVLX())) {
4432 0 : return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4433 : }
4434 : return 0;
4435 : }
4436 :
4437 : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4438 0 : if (RetVT.SimpleTy != MVT::v8f32)
4439 : return 0;
4440 0 : if ((Subtarget->hasVLX())) {
4441 0 : return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4442 : }
4443 : return 0;
4444 : }
4445 :
4446 : unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4447 0 : if (RetVT.SimpleTy != MVT::v16f32)
4448 : return 0;
4449 0 : if ((Subtarget->hasAVX512())) {
4450 0 : return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
4451 : }
4452 : return 0;
4453 : }
4454 :
4455 : unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4456 0 : if (RetVT.SimpleTy != MVT::v2f64)
4457 : return 0;
4458 0 : if ((Subtarget->hasVLX())) {
4459 0 : return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4460 : }
4461 : return 0;
4462 : }
4463 :
4464 : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4465 0 : if (RetVT.SimpleTy != MVT::v4f64)
4466 : return 0;
4467 0 : if ((Subtarget->hasVLX())) {
4468 0 : return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4469 : }
4470 : return 0;
4471 : }
4472 :
4473 : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4474 0 : if (RetVT.SimpleTy != MVT::v8f64)
4475 : return 0;
4476 0 : if ((Subtarget->hasAVX512())) {
4477 0 : return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4478 : }
4479 : return 0;
4480 : }
4481 :
4482 0 : unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4483 0 : switch (VT.SimpleTy) {
4484 0 : case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4485 0 : case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4486 0 : case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4487 0 : case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4488 0 : case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4489 0 : case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4490 : default: return 0;
4491 : }
4492 : }
4493 :
4494 : // FastEmit functions for X86ISD::SEG_ALLOCA.
4495 :
4496 0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4497 0 : if (RetVT.SimpleTy != MVT::i32)
4498 : return 0;
4499 0 : if ((!Subtarget->isTarget64BitLP64())) {
4500 0 : return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
4501 : }
4502 : return 0;
4503 : }
4504 :
4505 : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4506 0 : if (RetVT.SimpleTy != MVT::i64)
4507 : return 0;
4508 0 : if ((Subtarget->is64Bit())) {
4509 0 : return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
4510 : }
4511 : return 0;
4512 : }
4513 :
4514 0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4515 0 : switch (VT.SimpleTy) {
4516 0 : case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
4517 0 : case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
4518 : default: return 0;
4519 : }
4520 : }
4521 :
4522 : // FastEmit functions for X86ISD::VBROADCAST.
4523 :
4524 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
4525 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4526 0 : return fastEmitInst_r(X86::VPBROADCASTDrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4527 : }
4528 : return 0;
4529 : }
4530 :
4531 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4532 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4533 0 : return fastEmitInst_r(X86::VPBROADCASTDrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4534 : }
4535 : return 0;
4536 : }
4537 :
4538 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
4539 0 : if ((Subtarget->hasAVX512())) {
4540 0 : return fastEmitInst_r(X86::VPBROADCASTDrZr, &X86::VR512RegClass, Op0, Op0IsKill);
4541 : }
4542 : return 0;
4543 : }
4544 :
4545 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4546 0 : switch (RetVT.SimpleTy) {
4547 0 : case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
4548 0 : case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill);
4549 0 : case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill);
4550 : default: return 0;
4551 : }
4552 : }
4553 :
4554 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
4555 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4556 0 : return fastEmitInst_r(X86::VPBROADCASTQrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4557 : }
4558 : return 0;
4559 : }
4560 :
4561 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
4562 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4563 0 : return fastEmitInst_r(X86::VPBROADCASTQrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4564 : }
4565 : return 0;
4566 : }
4567 :
4568 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4569 0 : if ((Subtarget->hasAVX512())) {
4570 0 : return fastEmitInst_r(X86::VPBROADCASTQrZr, &X86::VR512RegClass, Op0, Op0IsKill);
4571 : }
4572 : return 0;
4573 : }
4574 :
4575 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4576 0 : switch (RetVT.SimpleTy) {
4577 0 : case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
4578 0 : case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill);
4579 0 : case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill);
4580 : default: return 0;
4581 : }
4582 : }
4583 :
4584 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
4585 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4586 0 : return fastEmitInst_r(X86::VPBROADCASTBZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4587 : }
4588 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4589 0 : return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill);
4590 : }
4591 : return 0;
4592 : }
4593 :
4594 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
4595 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4596 0 : return fastEmitInst_r(X86::VPBROADCASTBZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4597 : }
4598 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4599 0 : return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4600 : }
4601 : return 0;
4602 : }
4603 :
4604 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) {
4605 0 : if ((Subtarget->hasBWI())) {
4606 0 : return fastEmitInst_r(X86::VPBROADCASTBZr, &X86::VR512RegClass, Op0, Op0IsKill);
4607 : }
4608 : return 0;
4609 : }
4610 :
4611 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4612 0 : switch (RetVT.SimpleTy) {
4613 0 : case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill);
4614 0 : case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill);
4615 0 : case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill);
4616 : default: return 0;
4617 : }
4618 : }
4619 :
4620 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
4621 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4622 0 : return fastEmitInst_r(X86::VPBROADCASTWZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4623 : }
4624 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4625 0 : return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill);
4626 : }
4627 : return 0;
4628 : }
4629 :
4630 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
4631 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4632 0 : return fastEmitInst_r(X86::VPBROADCASTWZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4633 : }
4634 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4635 0 : return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4636 : }
4637 : return 0;
4638 : }
4639 :
4640 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
4641 0 : if ((Subtarget->hasBWI())) {
4642 0 : return fastEmitInst_r(X86::VPBROADCASTWZr, &X86::VR512RegClass, Op0, Op0IsKill);
4643 : }
4644 : return 0;
4645 : }
4646 :
4647 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4648 0 : switch (RetVT.SimpleTy) {
4649 0 : case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill);
4650 0 : case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill);
4651 0 : case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill);
4652 : default: return 0;
4653 : }
4654 : }
4655 :
4656 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
4657 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4658 0 : return fastEmitInst_r(X86::VPBROADCASTDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4659 : }
4660 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4661 0 : return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill);
4662 : }
4663 : return 0;
4664 : }
4665 :
4666 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4667 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4668 0 : return fastEmitInst_r(X86::VPBROADCASTDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4669 : }
4670 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4671 0 : return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4672 : }
4673 : return 0;
4674 : }
4675 :
4676 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
4677 0 : if ((Subtarget->hasAVX512())) {
4678 0 : return fastEmitInst_r(X86::VPBROADCASTDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4679 : }
4680 : return 0;
4681 : }
4682 :
4683 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4684 0 : switch (RetVT.SimpleTy) {
4685 0 : case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill);
4686 0 : case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill);
4687 0 : case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill);
4688 : default: return 0;
4689 : }
4690 : }
4691 :
4692 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
4693 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4694 0 : return fastEmitInst_r(X86::VPBROADCASTQZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4695 : }
4696 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4697 0 : return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill);
4698 : }
4699 : return 0;
4700 : }
4701 :
4702 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
4703 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4704 0 : return fastEmitInst_r(X86::VPBROADCASTQZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4705 : }
4706 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4707 0 : return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4708 : }
4709 : return 0;
4710 : }
4711 :
4712 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4713 0 : if ((Subtarget->hasAVX512())) {
4714 0 : return fastEmitInst_r(X86::VPBROADCASTQZr, &X86::VR512RegClass, Op0, Op0IsKill);
4715 : }
4716 : return 0;
4717 : }
4718 :
4719 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4720 0 : switch (RetVT.SimpleTy) {
4721 0 : case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill);
4722 0 : case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill);
4723 0 : case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill);
4724 : default: return 0;
4725 : }
4726 : }
4727 :
4728 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
4729 0 : if ((Subtarget->hasVLX())) {
4730 0 : return fastEmitInst_r(X86::VBROADCASTSSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4731 : }
4732 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4733 0 : return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill);
4734 : }
4735 : return 0;
4736 : }
4737 :
4738 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
4739 0 : if ((Subtarget->hasVLX())) {
4740 0 : return fastEmitInst_r(X86::VBROADCASTSSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4741 : }
4742 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4743 0 : return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4744 : }
4745 : return 0;
4746 : }
4747 :
4748 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) {
4749 0 : if ((Subtarget->hasAVX512())) {
4750 0 : return fastEmitInst_r(X86::VBROADCASTSSZr, &X86::VR512RegClass, Op0, Op0IsKill);
4751 : }
4752 : return 0;
4753 : }
4754 :
4755 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4756 0 : switch (RetVT.SimpleTy) {
4757 0 : case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill);
4758 0 : case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill);
4759 0 : case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill);
4760 : default: return 0;
4761 : }
4762 : }
4763 :
4764 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
4765 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4766 0 : return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4767 : }
4768 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4769 0 : return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4770 : }
4771 : return 0;
4772 : }
4773 :
4774 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
4775 0 : if ((Subtarget->hasVLX())) {
4776 0 : return fastEmitInst_r(X86::VBROADCASTSDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4777 : }
4778 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4779 0 : return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4780 : }
4781 : return 0;
4782 : }
4783 :
4784 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
4785 0 : if ((Subtarget->hasAVX512())) {
4786 0 : return fastEmitInst_r(X86::VBROADCASTSDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4787 : }
4788 : return 0;
4789 : }
4790 :
4791 0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4792 0 : switch (RetVT.SimpleTy) {
4793 0 : case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0, Op0IsKill);
4794 0 : case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill);
4795 0 : case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill);
4796 : default: return 0;
4797 : }
4798 : }
4799 :
4800 0 : unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4801 0 : switch (VT.SimpleTy) {
4802 0 : case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
4803 0 : case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
4804 0 : case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
4805 0 : case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
4806 0 : case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4807 0 : case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4808 0 : case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4809 0 : case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4810 : default: return 0;
4811 : }
4812 : }
4813 :
4814 : // FastEmit functions for X86ISD::VBROADCASTM.
4815 :
4816 0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
4817 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
4818 0 : return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4819 : }
4820 : return 0;
4821 : }
4822 :
4823 0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
4824 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
4825 0 : return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4826 : }
4827 : return 0;
4828 : }
4829 :
4830 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4831 0 : if ((Subtarget->hasCDI())) {
4832 0 : return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4833 : }
4834 : return 0;
4835 : }
4836 :
4837 0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4838 0 : switch (RetVT.SimpleTy) {
4839 0 : case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill);
4840 0 : case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill);
4841 0 : case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
4842 : default: return 0;
4843 : }
4844 : }
4845 :
4846 0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
4847 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
4848 0 : return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4849 : }
4850 : return 0;
4851 : }
4852 :
4853 0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4854 0 : if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
4855 0 : return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4856 : }
4857 : return 0;
4858 : }
4859 :
4860 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
4861 0 : if ((Subtarget->hasCDI())) {
4862 0 : return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4863 : }
4864 : return 0;
4865 : }
4866 :
4867 0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4868 0 : switch (RetVT.SimpleTy) {
4869 0 : case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill);
4870 0 : case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill);
4871 0 : case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
4872 : default: return 0;
4873 : }
4874 : }
4875 :
4876 0 : unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4877 0 : switch (VT.SimpleTy) {
4878 0 : case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
4879 0 : case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
4880 : default: return 0;
4881 : }
4882 : }
4883 :
4884 : // FastEmit functions for X86ISD::VFPEXT.
4885 :
4886 0 : unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4887 0 : if (RetVT.SimpleTy != MVT::v2f64)
4888 : return 0;
4889 0 : if ((Subtarget->hasVLX())) {
4890 0 : return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4891 : }
4892 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4893 0 : return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
4894 : }
4895 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4896 0 : return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
4897 : }
4898 : return 0;
4899 : }
4900 :
4901 : unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4902 0 : switch (VT.SimpleTy) {
4903 0 : case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4904 : default: return 0;
4905 : }
4906 : }
4907 :
4908 : // FastEmit functions for X86ISD::VFPROUND.
4909 :
4910 0 : unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4911 0 : if (RetVT.SimpleTy != MVT::v4f32)
4912 : return 0;
4913 0 : if ((Subtarget->hasVLX())) {
4914 0 : return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4915 : }
4916 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4917 0 : return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
4918 : }
4919 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4920 0 : return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
4921 : }
4922 : return 0;
4923 : }
4924 :
4925 : unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4926 0 : switch (VT.SimpleTy) {
4927 0 : case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4928 : default: return 0;
4929 : }
4930 : }
4931 :
4932 : // FastEmit functions for X86ISD::VSEXT.
4933 :
4934 0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
4935 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4936 0 : return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4937 : }
4938 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4939 0 : return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4940 : }
4941 : return 0;
4942 : }
4943 :
4944 0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4945 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4946 0 : return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4947 : }
4948 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4949 0 : return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4950 : }
4951 : return 0;
4952 : }
4953 :
4954 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
4955 0 : if ((Subtarget->hasAVX512())) {
4956 0 : return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4957 : }
4958 : return 0;
4959 : }
4960 :
4961 0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
4962 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4963 0 : return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4964 : }
4965 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4966 0 : return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4967 : }
4968 : return 0;
4969 : }
4970 :
4971 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4972 0 : if ((Subtarget->hasAVX512())) {
4973 0 : return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4974 : }
4975 : return 0;
4976 : }
4977 :
4978 0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4979 0 : switch (RetVT.SimpleTy) {
4980 0 : case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
4981 0 : case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
4982 0 : case MVT::v16i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
4983 0 : case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
4984 0 : case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
4985 : default: return 0;
4986 : }
4987 : }
4988 :
4989 : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4990 0 : if (RetVT.SimpleTy != MVT::v32i16)
4991 : return 0;
4992 0 : if ((Subtarget->hasBWI())) {
4993 0 : return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4994 : }
4995 : return 0;
4996 : }
4997 :
4998 0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4999 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5000 0 : return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5001 : }
5002 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5003 0 : return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5004 : }
5005 : return 0;
5006 : }
5007 :
5008 0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
5009 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5010 0 : return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5011 : }
5012 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5013 0 : return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5014 : }
5015 : return 0;
5016 : }
5017 :
5018 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
5019 0 : if ((Subtarget->hasAVX512())) {
5020 0 : return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5021 : }
5022 : return 0;
5023 : }
5024 :
5025 0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5026 0 : switch (RetVT.SimpleTy) {
5027 0 : case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
5028 0 : case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
5029 0 : case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
5030 : default: return 0;
5031 : }
5032 : }
5033 :
5034 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5035 0 : if (RetVT.SimpleTy != MVT::v16i32)
5036 : return 0;
5037 0 : if ((Subtarget->hasAVX512())) {
5038 0 : return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5039 : }
5040 : return 0;
5041 : }
5042 :
5043 0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5044 0 : if (RetVT.SimpleTy != MVT::v4i64)
5045 : return 0;
5046 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5047 0 : return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5048 : }
5049 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5050 0 : return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5051 : }
5052 : return 0;
5053 : }
5054 :
5055 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5056 0 : if (RetVT.SimpleTy != MVT::v8i64)
5057 : return 0;
5058 0 : if ((Subtarget->hasAVX512())) {
5059 0 : return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5060 : }
5061 : return 0;
5062 : }
5063 :
5064 0 : unsigned fastEmit_X86ISD_VSEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5065 0 : switch (VT.SimpleTy) {
5066 0 : case MVT::v16i8: return fastEmit_X86ISD_VSEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
5067 0 : case MVT::v32i8: return fastEmit_X86ISD_VSEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
5068 0 : case MVT::v8i16: return fastEmit_X86ISD_VSEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
5069 0 : case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
5070 0 : case MVT::v4i32: return fastEmit_X86ISD_VSEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
5071 0 : case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
5072 : default: return 0;
5073 : }
5074 : }
5075 :
5076 : // FastEmit functions for X86ISD::VTRUNC.
5077 :
5078 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5079 0 : if (RetVT.SimpleTy != MVT::v16i8)
5080 : return 0;
5081 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5082 0 : return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5083 : }
5084 : return 0;
5085 : }
5086 :
5087 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5088 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5089 0 : return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5090 : }
5091 : return 0;
5092 : }
5093 :
5094 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5095 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5096 0 : return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5097 : }
5098 : return 0;
5099 : }
5100 :
5101 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5102 0 : switch (RetVT.SimpleTy) {
5103 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
5104 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
5105 : default: return 0;
5106 : }
5107 : }
5108 :
5109 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5110 0 : if (RetVT.SimpleTy != MVT::v16i8)
5111 : return 0;
5112 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5113 0 : return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5114 : }
5115 : return 0;
5116 : }
5117 :
5118 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5119 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5120 0 : return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5121 : }
5122 : return 0;
5123 : }
5124 :
5125 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5126 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5127 0 : return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5128 : }
5129 : return 0;
5130 : }
5131 :
5132 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5133 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5134 0 : return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5135 : }
5136 : return 0;
5137 : }
5138 :
5139 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5140 0 : switch (RetVT.SimpleTy) {
5141 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
5142 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
5143 0 : case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
5144 : default: return 0;
5145 : }
5146 : }
5147 :
5148 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5149 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5150 0 : return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5151 : }
5152 : return 0;
5153 : }
5154 :
5155 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5156 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5157 0 : return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5158 : }
5159 : return 0;
5160 : }
5161 :
5162 0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5163 0 : switch (RetVT.SimpleTy) {
5164 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
5165 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
5166 : default: return 0;
5167 : }
5168 : }
5169 :
5170 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5171 0 : if (RetVT.SimpleTy != MVT::v16i8)
5172 : return 0;
5173 0 : if ((Subtarget->hasAVX512())) {
5174 0 : return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5175 : }
5176 : return 0;
5177 : }
5178 :
5179 0 : unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5180 0 : switch (VT.SimpleTy) {
5181 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
5182 0 : case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
5183 0 : case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
5184 0 : case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
5185 0 : case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
5186 0 : case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
5187 : default: return 0;
5188 : }
5189 : }
5190 :
5191 : // FastEmit functions for X86ISD::VTRUNCS.
5192 :
5193 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5194 0 : if (RetVT.SimpleTy != MVT::v16i8)
5195 : return 0;
5196 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5197 0 : return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5198 : }
5199 : return 0;
5200 : }
5201 :
5202 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5203 0 : if (RetVT.SimpleTy != MVT::v16i8)
5204 : return 0;
5205 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5206 0 : return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5207 : }
5208 : return 0;
5209 : }
5210 :
5211 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5212 0 : if (RetVT.SimpleTy != MVT::v32i8)
5213 : return 0;
5214 0 : if ((Subtarget->hasBWI())) {
5215 0 : return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5216 : }
5217 : return 0;
5218 : }
5219 :
5220 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5221 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5222 0 : return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5223 : }
5224 : return 0;
5225 : }
5226 :
5227 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5228 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5229 0 : return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5230 : }
5231 : return 0;
5232 : }
5233 :
5234 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5235 0 : switch (RetVT.SimpleTy) {
5236 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
5237 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
5238 : default: return 0;
5239 : }
5240 : }
5241 :
5242 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5243 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5244 0 : return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5245 : }
5246 : return 0;
5247 : }
5248 :
5249 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5250 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5251 0 : return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5252 : }
5253 : return 0;
5254 : }
5255 :
5256 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5257 0 : switch (RetVT.SimpleTy) {
5258 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
5259 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
5260 : default: return 0;
5261 : }
5262 : }
5263 :
5264 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5265 0 : if ((Subtarget->hasAVX512())) {
5266 0 : return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5267 : }
5268 : return 0;
5269 : }
5270 :
5271 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
5272 0 : if ((Subtarget->hasAVX512())) {
5273 0 : return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5274 : }
5275 : return 0;
5276 : }
5277 :
5278 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5279 0 : switch (RetVT.SimpleTy) {
5280 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
5281 0 : case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
5282 : default: return 0;
5283 : }
5284 : }
5285 :
5286 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5287 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5288 0 : return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5289 : }
5290 : return 0;
5291 : }
5292 :
5293 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5294 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5295 0 : return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5296 : }
5297 : return 0;
5298 : }
5299 :
5300 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5301 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5302 0 : return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5303 : }
5304 : return 0;
5305 : }
5306 :
5307 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5308 0 : switch (RetVT.SimpleTy) {
5309 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
5310 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
5311 0 : case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
5312 : default: return 0;
5313 : }
5314 : }
5315 :
5316 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5317 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5318 0 : return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5319 : }
5320 : return 0;
5321 : }
5322 :
5323 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5324 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5325 0 : return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5326 : }
5327 : return 0;
5328 : }
5329 :
5330 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5331 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5332 0 : return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5333 : }
5334 : return 0;
5335 : }
5336 :
5337 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5338 0 : switch (RetVT.SimpleTy) {
5339 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
5340 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
5341 0 : case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
5342 : default: return 0;
5343 : }
5344 : }
5345 :
5346 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5347 0 : if ((Subtarget->hasAVX512())) {
5348 0 : return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5349 : }
5350 : return 0;
5351 : }
5352 :
5353 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5354 0 : if ((Subtarget->hasAVX512())) {
5355 0 : return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5356 : }
5357 : return 0;
5358 : }
5359 :
5360 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5361 0 : if ((Subtarget->hasAVX512())) {
5362 0 : return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5363 : }
5364 : return 0;
5365 : }
5366 :
5367 0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5368 0 : switch (RetVT.SimpleTy) {
5369 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
5370 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
5371 0 : case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
5372 : default: return 0;
5373 : }
5374 : }
5375 :
5376 0 : unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5377 0 : switch (VT.SimpleTy) {
5378 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
5379 0 : case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
5380 0 : case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
5381 0 : case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
5382 0 : case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
5383 0 : case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
5384 0 : case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
5385 0 : case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
5386 0 : case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
5387 : default: return 0;
5388 : }
5389 : }
5390 :
5391 : // FastEmit functions for X86ISD::VTRUNCUS.
5392 :
5393 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5394 0 : if (RetVT.SimpleTy != MVT::v16i8)
5395 : return 0;
5396 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5397 0 : return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5398 : }
5399 : return 0;
5400 : }
5401 :
5402 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5403 0 : if (RetVT.SimpleTy != MVT::v16i8)
5404 : return 0;
5405 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5406 0 : return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5407 : }
5408 : return 0;
5409 : }
5410 :
5411 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5412 0 : if (RetVT.SimpleTy != MVT::v32i8)
5413 : return 0;
5414 0 : if ((Subtarget->hasBWI())) {
5415 0 : return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5416 : }
5417 : return 0;
5418 : }
5419 :
5420 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5421 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5422 0 : return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5423 : }
5424 : return 0;
5425 : }
5426 :
5427 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5428 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5429 0 : return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5430 : }
5431 : return 0;
5432 : }
5433 :
5434 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5435 0 : switch (RetVT.SimpleTy) {
5436 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
5437 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
5438 : default: return 0;
5439 : }
5440 : }
5441 :
5442 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5443 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5444 0 : return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5445 : }
5446 : return 0;
5447 : }
5448 :
5449 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5450 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5451 0 : return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5452 : }
5453 : return 0;
5454 : }
5455 :
5456 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5457 0 : switch (RetVT.SimpleTy) {
5458 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
5459 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
5460 : default: return 0;
5461 : }
5462 : }
5463 :
5464 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5465 0 : if ((Subtarget->hasAVX512())) {
5466 0 : return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5467 : }
5468 : return 0;
5469 : }
5470 :
5471 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
5472 0 : if ((Subtarget->hasAVX512())) {
5473 0 : return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5474 : }
5475 : return 0;
5476 : }
5477 :
5478 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5479 0 : switch (RetVT.SimpleTy) {
5480 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
5481 0 : case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
5482 : default: return 0;
5483 : }
5484 : }
5485 :
5486 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5487 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5488 0 : return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5489 : }
5490 : return 0;
5491 : }
5492 :
5493 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5494 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5495 0 : return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5496 : }
5497 : return 0;
5498 : }
5499 :
5500 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5501 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5502 0 : return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5503 : }
5504 : return 0;
5505 : }
5506 :
5507 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5508 0 : switch (RetVT.SimpleTy) {
5509 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
5510 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
5511 0 : case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
5512 : default: return 0;
5513 : }
5514 : }
5515 :
5516 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5517 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5518 0 : return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5519 : }
5520 : return 0;
5521 : }
5522 :
5523 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5524 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5525 0 : return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5526 : }
5527 : return 0;
5528 : }
5529 :
5530 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5531 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5532 0 : return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5533 : }
5534 : return 0;
5535 : }
5536 :
5537 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5538 0 : switch (RetVT.SimpleTy) {
5539 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
5540 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
5541 0 : case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
5542 : default: return 0;
5543 : }
5544 : }
5545 :
5546 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5547 0 : if ((Subtarget->hasAVX512())) {
5548 0 : return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5549 : }
5550 : return 0;
5551 : }
5552 :
5553 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5554 0 : if ((Subtarget->hasAVX512())) {
5555 0 : return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5556 : }
5557 : return 0;
5558 : }
5559 :
5560 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5561 0 : if ((Subtarget->hasAVX512())) {
5562 0 : return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5563 : }
5564 : return 0;
5565 : }
5566 :
5567 0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5568 0 : switch (RetVT.SimpleTy) {
5569 0 : case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
5570 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
5571 0 : case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
5572 : default: return 0;
5573 : }
5574 : }
5575 :
5576 0 : unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5577 0 : switch (VT.SimpleTy) {
5578 0 : case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
5579 0 : case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
5580 0 : case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
5581 0 : case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
5582 0 : case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
5583 0 : case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
5584 0 : case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
5585 0 : case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
5586 0 : case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
5587 : default: return 0;
5588 : }
5589 : }
5590 :
5591 : // FastEmit functions for X86ISD::VZEXT.
5592 :
5593 0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
5594 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5595 0 : return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5596 : }
5597 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5598 0 : return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5599 : }
5600 : return 0;
5601 : }
5602 :
5603 0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5604 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5605 0 : return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5606 : }
5607 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5608 0 : return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5609 : }
5610 : return 0;
5611 : }
5612 :
5613 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
5614 0 : if ((Subtarget->hasAVX512())) {
5615 0 : return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5616 : }
5617 : return 0;
5618 : }
5619 :
5620 0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
5621 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5622 0 : return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5623 : }
5624 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5625 0 : return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5626 : }
5627 : return 0;
5628 : }
5629 :
5630 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
5631 0 : if ((Subtarget->hasAVX512())) {
5632 0 : return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5633 : }
5634 : return 0;
5635 : }
5636 :
5637 0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5638 0 : switch (RetVT.SimpleTy) {
5639 0 : case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
5640 0 : case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
5641 0 : case MVT::v16i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
5642 0 : case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
5643 0 : case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
5644 : default: return 0;
5645 : }
5646 : }
5647 :
5648 : unsigned fastEmit_X86ISD_VZEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5649 0 : if (RetVT.SimpleTy != MVT::v32i16)
5650 : return 0;
5651 0 : if ((Subtarget->hasBWI())) {
5652 0 : return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5653 : }
5654 : return 0;
5655 : }
5656 :
5657 0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5658 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5659 0 : return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5660 : }
5661 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5662 0 : return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5663 : }
5664 : return 0;
5665 : }
5666 :
5667 0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
5668 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5669 0 : return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5670 : }
5671 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5672 0 : return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5673 : }
5674 : return 0;
5675 : }
5676 :
5677 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
5678 0 : if ((Subtarget->hasAVX512())) {
5679 0 : return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5680 : }
5681 : return 0;
5682 : }
5683 :
5684 0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5685 0 : switch (RetVT.SimpleTy) {
5686 0 : case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
5687 0 : case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
5688 0 : case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
5689 : default: return 0;
5690 : }
5691 : }
5692 :
5693 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5694 0 : if (RetVT.SimpleTy != MVT::v16i32)
5695 : return 0;
5696 0 : if ((Subtarget->hasAVX512())) {
5697 0 : return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5698 : }
5699 : return 0;
5700 : }
5701 :
5702 0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5703 0 : if (RetVT.SimpleTy != MVT::v4i64)
5704 : return 0;
5705 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5706 0 : return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5707 : }
5708 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5709 0 : return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5710 : }
5711 : return 0;
5712 : }
5713 :
5714 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5715 0 : if (RetVT.SimpleTy != MVT::v8i64)
5716 : return 0;
5717 0 : if ((Subtarget->hasAVX512())) {
5718 0 : return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5719 : }
5720 : return 0;
5721 : }
5722 :
5723 0 : unsigned fastEmit_X86ISD_VZEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5724 0 : switch (VT.SimpleTy) {
5725 0 : case MVT::v16i8: return fastEmit_X86ISD_VZEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
5726 0 : case MVT::v32i8: return fastEmit_X86ISD_VZEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
5727 0 : case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
5728 0 : case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
5729 0 : case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
5730 0 : case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
5731 : default: return 0;
5732 : }
5733 : }
5734 :
5735 : // FastEmit functions for X86ISD::VZEXT_MOVL.
5736 :
5737 0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5738 0 : if (RetVT.SimpleTy != MVT::v2i64)
5739 : return 0;
5740 0 : if ((Subtarget->hasAVX512())) {
5741 0 : return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5742 : }
5743 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5744 0 : return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
5745 : }
5746 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5747 0 : return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
5748 : }
5749 : return 0;
5750 : }
5751 :
5752 0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5753 0 : if (RetVT.SimpleTy != MVT::v2f64)
5754 : return 0;
5755 0 : if ((Subtarget->hasAVX512())) {
5756 0 : return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
5757 : }
5758 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5759 0 : return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
5760 : }
5761 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
5762 0 : return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
5763 : }
5764 : return 0;
5765 : }
5766 :
5767 0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5768 0 : switch (VT.SimpleTy) {
5769 0 : case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
5770 0 : case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
5771 : default: return 0;
5772 : }
5773 : }
5774 :
5775 : // FastEmit functions for X86ISD::WIN_ALLOCA.
5776 :
5777 0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5778 0 : if (RetVT.SimpleTy != MVT::isVoid)
5779 : return 0;
5780 0 : if ((!Subtarget->isTarget64BitLP64())) {
5781 0 : return fastEmitInst_r(X86::WIN_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
5782 : }
5783 : return 0;
5784 : }
5785 :
5786 : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5787 0 : if (RetVT.SimpleTy != MVT::isVoid)
5788 : return 0;
5789 0 : if ((Subtarget->is64Bit())) {
5790 0 : return fastEmitInst_r(X86::WIN_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
5791 : }
5792 : return 0;
5793 : }
5794 :
5795 0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5796 0 : switch (VT.SimpleTy) {
5797 0 : case MVT::i32: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
5798 0 : case MVT::i64: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
5799 : default: return 0;
5800 : }
5801 : }
5802 :
5803 : // Top-level FastEmit function.
5804 :
5805 203121 : unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
5806 203121 : switch (Opcode) {
5807 0 : case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
5808 36 : case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
5809 4267 : case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
5810 0 : case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
5811 0 : case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
5812 0 : case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
5813 0 : case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
5814 0 : case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0, Op0IsKill);
5815 0 : case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0, Op0IsKill);
5816 0 : case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
5817 0 : case ISD::FCOS: return fastEmit_ISD_FCOS_r(VT, RetVT, Op0, Op0IsKill);
5818 188 : case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
5819 0 : case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
5820 0 : case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
5821 120 : case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
5822 0 : case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
5823 0 : case ISD::FSIN: return fastEmit_ISD_FSIN_r(VT, RetVT, Op0, Op0IsKill);
5824 0 : case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
5825 1 : case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0, Op0IsKill);
5826 56366 : case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
5827 0 : case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
5828 1101 : case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
5829 12880 : case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
5830 0 : case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
5831 128162 : case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
5832 0 : case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
5833 0 : case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
5834 0 : case X86ISD::COMPRESS: return fastEmit_X86ISD_COMPRESS_r(VT, RetVT, Op0, Op0IsKill);
5835 0 : case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0, Op0IsKill);
5836 0 : case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0, Op0IsKill);
5837 0 : case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0, Op0IsKill);
5838 0 : case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0, Op0IsKill);
5839 0 : case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0, Op0IsKill);
5840 0 : case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0, Op0IsKill);
5841 0 : case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill);
5842 0 : case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill);
5843 0 : case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill);
5844 0 : case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0, Op0IsKill);
5845 0 : case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0, Op0IsKill);
5846 0 : case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill);
5847 0 : case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0, Op0IsKill);
5848 0 : case X86ISD::EXPAND: return fastEmit_X86ISD_EXPAND_r(VT, RetVT, Op0, Op0IsKill);
5849 0 : case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0, Op0IsKill);
5850 0 : case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0, Op0IsKill);
5851 0 : case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0, Op0IsKill);
5852 0 : case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0, Op0IsKill);
5853 0 : case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0, Op0IsKill);
5854 0 : case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0, Op0IsKill);
5855 0 : case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0, Op0IsKill);
5856 0 : case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0, Op0IsKill);
5857 0 : case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0, Op0IsKill);
5858 0 : case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0, Op0IsKill);
5859 0 : case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0, Op0IsKill);
5860 0 : case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0, Op0IsKill);
5861 0 : case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0, Op0IsKill);
5862 0 : case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0, Op0IsKill);
5863 0 : case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
5864 0 : case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0, Op0IsKill);
5865 0 : case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0, Op0IsKill);
5866 0 : case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0, Op0IsKill);
5867 0 : case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0, Op0IsKill);
5868 0 : case X86ISD::VSEXT: return fastEmit_X86ISD_VSEXT_r(VT, RetVT, Op0, Op0IsKill);
5869 0 : case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0, Op0IsKill);
5870 0 : case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0, Op0IsKill);
5871 0 : case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0, Op0IsKill);
5872 0 : case X86ISD::VZEXT: return fastEmit_X86ISD_VZEXT_r(VT, RetVT, Op0, Op0IsKill);
5873 0 : case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0, Op0IsKill);
5874 0 : case X86ISD::WIN_ALLOCA: return fastEmit_X86ISD_WIN_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
5875 : default: return 0;
5876 : }
5877 : }
5878 :
5879 : // FastEmit functions for ISD::ADD.
5880 :
5881 : unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5882 1 : if (RetVT.SimpleTy != MVT::i8)
5883 : return 0;
5884 1 : return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5885 : }
5886 :
5887 : unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5888 1 : if (RetVT.SimpleTy != MVT::i16)
5889 : return 0;
5890 1 : return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5891 : }
5892 :
5893 : unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5894 2683 : if (RetVT.SimpleTy != MVT::i32)
5895 : return 0;
5896 2683 : return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5897 : }
5898 :
5899 : unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5900 80918 : if (RetVT.SimpleTy != MVT::i64)
5901 : return 0;
5902 80918 : return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5903 : }
5904 :
5905 6 : unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5906 6 : if (RetVT.SimpleTy != MVT::v16i8)
5907 : return 0;
5908 6 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5909 2 : return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5910 : }
5911 8 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5912 2 : return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5913 : }
5914 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5915 2 : return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5916 : }
5917 : return 0;
5918 : }
5919 :
5920 2 : unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5921 2 : if (RetVT.SimpleTy != MVT::v32i8)
5922 : return 0;
5923 2 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5924 0 : return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5925 : }
5926 4 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5927 2 : return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5928 : }
5929 : return 0;
5930 : }
5931 :
5932 : unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5933 0 : if (RetVT.SimpleTy != MVT::v64i8)
5934 : return 0;
5935 0 : if ((Subtarget->hasBWI())) {
5936 0 : return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5937 : }
5938 : return 0;
5939 : }
5940 :
5941 6 : unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5942 6 : if (RetVT.SimpleTy != MVT::v8i16)
5943 : return 0;
5944 6 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5945 2 : return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5946 : }
5947 8 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5948 2 : return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5949 : }
5950 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5951 2 : return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5952 : }
5953 : return 0;
5954 : }
5955 :
5956 2 : unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5957 2 : if (RetVT.SimpleTy != MVT::v16i16)
5958 : return 0;
5959 2 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5960 0 : return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5961 : }
5962 4 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5963 2 : return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5964 : }
5965 : return 0;
5966 : }
5967 :
5968 : unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5969 0 : if (RetVT.SimpleTy != MVT::v32i16)
5970 : return 0;
5971 0 : if ((Subtarget->hasBWI())) {
5972 0 : return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5973 : }
5974 : return 0;
5975 : }
5976 :
5977 22 : unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5978 22 : if (RetVT.SimpleTy != MVT::v4i32)
5979 : return 0;
5980 44 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5981 4 : return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5982 : }
5983 18 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5984 6 : return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5985 : }
5986 12 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5987 12 : return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5988 : }
5989 : return 0;
5990 : }
5991 :
5992 3 : unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5993 3 : if (RetVT.SimpleTy != MVT::v8i32)
5994 : return 0;
5995 6 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5996 1 : return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5997 : }
5998 2 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
5999 2 : return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6000 : }
6001 : return 0;
6002 : }
6003 :
6004 : unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6005 2 : if (RetVT.SimpleTy != MVT::v16i32)
6006 : return 0;
6007 4 : if ((Subtarget->hasAVX512())) {
6008 2 : return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6009 : }
6010 : return 0;
6011 : }
6012 :
6013 6 : unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6014 6 : if (RetVT.SimpleTy != MVT::v2i64)
6015 : return 0;
6016 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6017 2 : return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6018 : }
6019 4 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6020 2 : return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6021 : }
6022 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6023 2 : return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6024 : }
6025 : return 0;
6026 : }
6027 :
6028 4 : unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6029 4 : if (RetVT.SimpleTy != MVT::v4i64)
6030 : return 0;
6031 8 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6032 2 : return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6033 : }
6034 2 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6035 2 : return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6036 : }
6037 : return 0;
6038 : }
6039 :
6040 : unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6041 0 : if (RetVT.SimpleTy != MVT::v8i64)
6042 : return 0;
6043 0 : if ((Subtarget->hasAVX512())) {
6044 0 : return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6045 : }
6046 : return 0;
6047 : }
6048 :
6049 83656 : unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6050 83656 : switch (VT.SimpleTy) {
6051 1 : case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6052 1 : case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6053 2683 : case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6054 80918 : case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6055 6 : case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6056 2 : case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6057 0 : case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6058 6 : case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6059 2 : case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6060 0 : case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6061 22 : case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6062 3 : case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6063 2 : case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6064 6 : case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6065 4 : case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6066 0 : case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6067 : default: return 0;
6068 : }
6069 : }
6070 :
6071 : // FastEmit functions for ISD::AND.
6072 :
6073 : unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6074 87 : if (RetVT.SimpleTy != MVT::i8)
6075 : return 0;
6076 87 : return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6077 : }
6078 :
6079 : unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6080 0 : if (RetVT.SimpleTy != MVT::i16)
6081 : return 0;
6082 0 : return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6083 : }
6084 :
6085 : unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6086 10966 : if (RetVT.SimpleTy != MVT::i32)
6087 : return 0;
6088 10966 : return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6089 : }
6090 :
6091 : unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6092 19846 : if (RetVT.SimpleTy != MVT::i64)
6093 : return 0;
6094 19846 : return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6095 : }
6096 :
6097 : unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6098 0 : if (RetVT.SimpleTy != MVT::v8i1)
6099 : return 0;
6100 0 : if ((Subtarget->hasDQI())) {
6101 0 : return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6102 : }
6103 : return 0;
6104 : }
6105 :
6106 : unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6107 0 : if (RetVT.SimpleTy != MVT::v16i1)
6108 : return 0;
6109 0 : if ((Subtarget->hasAVX512())) {
6110 0 : return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6111 : }
6112 : return 0;
6113 : }
6114 :
6115 : unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6116 0 : if (RetVT.SimpleTy != MVT::v32i1)
6117 : return 0;
6118 0 : if ((Subtarget->hasBWI())) {
6119 0 : return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6120 : }
6121 : return 0;
6122 : }
6123 :
6124 : unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6125 0 : if (RetVT.SimpleTy != MVT::v64i1)
6126 : return 0;
6127 0 : if ((Subtarget->hasBWI())) {
6128 0 : return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6129 : }
6130 : return 0;
6131 : }
6132 :
6133 16 : unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6134 16 : if (RetVT.SimpleTy != MVT::v2i64)
6135 : return 0;
6136 32 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6137 4 : return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6138 : }
6139 12 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6140 4 : return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6141 : }
6142 8 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6143 8 : return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6144 : }
6145 : return 0;
6146 : }
6147 :
6148 12 : unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6149 12 : if (RetVT.SimpleTy != MVT::v4i64)
6150 : return 0;
6151 24 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
6152 8 : return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6153 : }
6154 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6155 0 : return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6156 : }
6157 4 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6158 4 : return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6159 : }
6160 : return 0;
6161 : }
6162 :
6163 : unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6164 0 : if (RetVT.SimpleTy != MVT::v8i64)
6165 : return 0;
6166 0 : if ((Subtarget->hasAVX512())) {
6167 0 : return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6168 : }
6169 : return 0;
6170 : }
6171 :
6172 30951 : unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6173 30951 : switch (VT.SimpleTy) {
6174 87 : case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6175 0 : case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6176 10966 : case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6177 19846 : case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6178 0 : case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6179 0 : case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6180 0 : case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6181 0 : case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6182 16 : case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6183 12 : case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6184 0 : case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6185 : default: return 0;
6186 : }
6187 : }
6188 :
6189 : // FastEmit functions for ISD::FADD.
6190 :
6191 106 : unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6192 106 : if (RetVT.SimpleTy != MVT::f32)
6193 : return 0;
6194 212 : if ((Subtarget->hasAVX512())) {
6195 2 : return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6196 : }
6197 104 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6198 102 : return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6199 : }
6200 2 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6201 2 : return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6202 : }
6203 0 : if ((!Subtarget->hasSSE1())) {
6204 0 : return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6205 : }
6206 : return 0;
6207 : }
6208 :
6209 1709 : unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6210 1709 : if (RetVT.SimpleTy != MVT::f64)
6211 : return 0;
6212 3418 : if ((Subtarget->hasAVX512())) {
6213 2 : return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6214 : }
6215 1707 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6216 1705 : return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6217 : }
6218 2 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6219 2 : return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6220 : }
6221 0 : if ((!Subtarget->hasSSE2())) {
6222 0 : return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6223 : }
6224 : return 0;
6225 : }
6226 :
6227 : unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6228 0 : if (RetVT.SimpleTy != MVT::f80)
6229 : return 0;
6230 0 : return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6231 : }
6232 :
6233 6 : unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6234 6 : if (RetVT.SimpleTy != MVT::v4f32)
6235 : return 0;
6236 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6237 2 : return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6238 : }
6239 4 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6240 2 : return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6241 : }
6242 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6243 2 : return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6244 : }
6245 : return 0;
6246 : }
6247 :
6248 2 : unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6249 2 : if (RetVT.SimpleTy != MVT::v8f32)
6250 : return 0;
6251 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6252 0 : return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6253 : }
6254 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6255 2 : return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6256 : }
6257 : return 0;
6258 : }
6259 :
6260 : unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6261 1 : if (RetVT.SimpleTy != MVT::v16f32)
6262 : return 0;
6263 2 : if ((Subtarget->hasAVX512())) {
6264 1 : return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6265 : }
6266 : return 0;
6267 : }
6268 :
6269 22 : unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6270 22 : if (RetVT.SimpleTy != MVT::v2f64)
6271 : return 0;
6272 44 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6273 4 : return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6274 : }
6275 18 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6276 6 : return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6277 : }
6278 12 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6279 12 : return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6280 : }
6281 : return 0;
6282 : }
6283 :
6284 14 : unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6285 14 : if (RetVT.SimpleTy != MVT::v4f64)
6286 : return 0;
6287 28 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6288 2 : return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6289 : }
6290 12 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6291 12 : return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6292 : }
6293 : return 0;
6294 : }
6295 :
6296 : unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6297 8 : if (RetVT.SimpleTy != MVT::v8f64)
6298 : return 0;
6299 16 : if ((Subtarget->hasAVX512())) {
6300 8 : return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6301 : }
6302 : return 0;
6303 : }
6304 :
6305 1868 : unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6306 1868 : switch (VT.SimpleTy) {
6307 106 : case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6308 1709 : case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6309 0 : case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6310 6 : case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6311 2 : case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6312 1 : case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6313 22 : case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6314 14 : case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6315 8 : case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6316 : default: return 0;
6317 : }
6318 : }
6319 :
6320 : // FastEmit functions for ISD::FDIV.
6321 :
6322 679 : unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6323 679 : if (RetVT.SimpleTy != MVT::f32)
6324 : return 0;
6325 1358 : if ((Subtarget->hasAVX512())) {
6326 0 : return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6327 : }
6328 679 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6329 679 : return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6330 : }
6331 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6332 0 : return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6333 : }
6334 0 : if ((!Subtarget->hasSSE1())) {
6335 0 : return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6336 : }
6337 : return 0;
6338 : }
6339 :
6340 1615 : unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6341 1615 : if (RetVT.SimpleTy != MVT::f64)
6342 : return 0;
6343 3230 : if ((Subtarget->hasAVX512())) {
6344 0 : return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6345 : }
6346 1615 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6347 1615 : return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6348 : }
6349 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6350 0 : return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6351 : }
6352 0 : if ((!Subtarget->hasSSE2())) {
6353 0 : return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6354 : }
6355 : return 0;
6356 : }
6357 :
6358 : unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6359 0 : if (RetVT.SimpleTy != MVT::f80)
6360 : return 0;
6361 0 : return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6362 : }
6363 :
6364 6 : unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6365 6 : if (RetVT.SimpleTy != MVT::v4f32)
6366 : return 0;
6367 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6368 2 : return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6369 : }
6370 4 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6371 2 : return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6372 : }
6373 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6374 2 : return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6375 : }
6376 : return 0;
6377 : }
6378 :
6379 2 : unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6380 2 : if (RetVT.SimpleTy != MVT::v8f32)
6381 : return 0;
6382 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6383 0 : return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6384 : }
6385 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6386 2 : return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6387 : }
6388 : return 0;
6389 : }
6390 :
6391 : unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6392 0 : if (RetVT.SimpleTy != MVT::v16f32)
6393 : return 0;
6394 0 : if ((Subtarget->hasAVX512())) {
6395 0 : return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6396 : }
6397 : return 0;
6398 : }
6399 :
6400 6 : unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6401 6 : if (RetVT.SimpleTy != MVT::v2f64)
6402 : return 0;
6403 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6404 2 : return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6405 : }
6406 4 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6407 2 : return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6408 : }
6409 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6410 2 : return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6411 : }
6412 : return 0;
6413 : }
6414 :
6415 2 : unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6416 2 : if (RetVT.SimpleTy != MVT::v4f64)
6417 : return 0;
6418 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6419 0 : return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6420 : }
6421 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6422 2 : return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6423 : }
6424 : return 0;
6425 : }
6426 :
6427 : unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6428 0 : if (RetVT.SimpleTy != MVT::v8f64)
6429 : return 0;
6430 0 : if ((Subtarget->hasAVX512())) {
6431 0 : return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6432 : }
6433 : return 0;
6434 : }
6435 :
6436 2310 : unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6437 2310 : switch (VT.SimpleTy) {
6438 679 : case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6439 1615 : case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6440 0 : case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6441 6 : case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6442 2 : case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6443 0 : case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6444 6 : case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6445 2 : case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6446 0 : case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6447 : default: return 0;
6448 : }
6449 : }
6450 :
6451 : // FastEmit functions for ISD::FMUL.
6452 :
6453 1025 : unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6454 1025 : if (RetVT.SimpleTy != MVT::f32)
6455 : return 0;
6456 2050 : if ((Subtarget->hasAVX512())) {
6457 0 : return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6458 : }
6459 1025 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6460 1025 : return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6461 : }
6462 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6463 0 : return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6464 : }
6465 0 : if ((!Subtarget->hasSSE1())) {
6466 0 : return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6467 : }
6468 : return 0;
6469 : }
6470 :
6471 2495 : unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6472 2495 : if (RetVT.SimpleTy != MVT::f64)
6473 : return 0;
6474 4990 : if ((Subtarget->hasAVX512())) {
6475 0 : return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6476 : }
6477 2495 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6478 2495 : return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6479 : }
6480 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6481 0 : return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6482 : }
6483 0 : if ((!Subtarget->hasSSE2())) {
6484 0 : return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6485 : }
6486 : return 0;
6487 : }
6488 :
6489 : unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6490 0 : if (RetVT.SimpleTy != MVT::f80)
6491 : return 0;
6492 0 : return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6493 : }
6494 :
6495 6 : unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6496 6 : if (RetVT.SimpleTy != MVT::v4f32)
6497 : return 0;
6498 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6499 2 : return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6500 : }
6501 4 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6502 2 : return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6503 : }
6504 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6505 2 : return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6506 : }
6507 : return 0;
6508 : }
6509 :
6510 2 : unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6511 2 : if (RetVT.SimpleTy != MVT::v8f32)
6512 : return 0;
6513 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6514 0 : return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6515 : }
6516 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6517 2 : return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6518 : }
6519 : return 0;
6520 : }
6521 :
6522 : unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6523 0 : if (RetVT.SimpleTy != MVT::v16f32)
6524 : return 0;
6525 0 : if ((Subtarget->hasAVX512())) {
6526 0 : return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6527 : }
6528 : return 0;
6529 : }
6530 :
6531 6 : unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6532 6 : if (RetVT.SimpleTy != MVT::v2f64)
6533 : return 0;
6534 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6535 2 : return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6536 : }
6537 4 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6538 2 : return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6539 : }
6540 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6541 2 : return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6542 : }
6543 : return 0;
6544 : }
6545 :
6546 2 : unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6547 2 : if (RetVT.SimpleTy != MVT::v4f64)
6548 : return 0;
6549 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6550 0 : return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6551 : }
6552 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6553 2 : return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6554 : }
6555 : return 0;
6556 : }
6557 :
6558 : unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6559 0 : if (RetVT.SimpleTy != MVT::v8f64)
6560 : return 0;
6561 0 : if ((Subtarget->hasAVX512())) {
6562 0 : return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6563 : }
6564 : return 0;
6565 : }
6566 :
6567 3536 : unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6568 3536 : switch (VT.SimpleTy) {
6569 1025 : case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6570 2495 : case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6571 0 : case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6572 6 : case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6573 2 : case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6574 0 : case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6575 6 : case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6576 2 : case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6577 0 : case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6578 : default: return 0;
6579 : }
6580 : }
6581 :
6582 : // FastEmit functions for ISD::FSUB.
6583 :
6584 268 : unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6585 268 : if (RetVT.SimpleTy != MVT::f32)
6586 : return 0;
6587 536 : if ((Subtarget->hasAVX512())) {
6588 0 : return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6589 : }
6590 268 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6591 268 : return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6592 : }
6593 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6594 0 : return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6595 : }
6596 0 : if ((!Subtarget->hasSSE1())) {
6597 0 : return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6598 : }
6599 : return 0;
6600 : }
6601 :
6602 1744 : unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6603 1744 : if (RetVT.SimpleTy != MVT::f64)
6604 : return 0;
6605 3488 : if ((Subtarget->hasAVX512())) {
6606 0 : return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6607 : }
6608 1744 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6609 1744 : return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6610 : }
6611 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6612 0 : return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6613 : }
6614 0 : if ((!Subtarget->hasSSE2())) {
6615 0 : return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6616 : }
6617 : return 0;
6618 : }
6619 :
6620 : unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6621 0 : if (RetVT.SimpleTy != MVT::f80)
6622 : return 0;
6623 0 : return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6624 : }
6625 :
6626 6 : unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6627 6 : if (RetVT.SimpleTy != MVT::v4f32)
6628 : return 0;
6629 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6630 2 : return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6631 : }
6632 4 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
6633 2 : return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6634 : }
6635 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6636 2 : return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6637 : }
6638 : return 0;
6639 : }
6640 :
6641 2 : unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6642 2 : if (RetVT.SimpleTy != MVT::v8f32)
6643 : return 0;
6644 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6645 0 : return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6646 : }
6647 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6648 2 : return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6649 : }
6650 : return 0;
6651 : }
6652 :
6653 : unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6654 0 : if (RetVT.SimpleTy != MVT::v16f32)
6655 : return 0;
6656 0 : if ((Subtarget->hasAVX512())) {
6657 0 : return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6658 : }
6659 : return 0;
6660 : }
6661 :
6662 6 : unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6663 6 : if (RetVT.SimpleTy != MVT::v2f64)
6664 : return 0;
6665 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6666 2 : return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6667 : }
6668 4 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6669 2 : return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6670 : }
6671 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6672 2 : return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6673 : }
6674 : return 0;
6675 : }
6676 :
6677 2 : unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6678 2 : if (RetVT.SimpleTy != MVT::v4f64)
6679 : return 0;
6680 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6681 0 : return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6682 : }
6683 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6684 2 : return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6685 : }
6686 : return 0;
6687 : }
6688 :
6689 : unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6690 0 : if (RetVT.SimpleTy != MVT::v8f64)
6691 : return 0;
6692 0 : if ((Subtarget->hasAVX512())) {
6693 0 : return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6694 : }
6695 : return 0;
6696 : }
6697 :
6698 2028 : unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6699 2028 : switch (VT.SimpleTy) {
6700 268 : case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6701 1744 : case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6702 0 : case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6703 6 : case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6704 2 : case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6705 0 : case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6706 6 : case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6707 2 : case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6708 0 : case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6709 : default: return 0;
6710 : }
6711 : }
6712 :
6713 : // FastEmit functions for ISD::MUL.
6714 :
6715 2 : unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6716 2 : if (RetVT.SimpleTy != MVT::i8)
6717 : return 0;
6718 4 : BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0);
6719 2 : return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1, Op1IsKill);
6720 : }
6721 :
6722 : unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6723 0 : if (RetVT.SimpleTy != MVT::i16)
6724 : return 0;
6725 0 : return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6726 : }
6727 :
6728 : unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6729 357 : if (RetVT.SimpleTy != MVT::i32)
6730 : return 0;
6731 357 : return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6732 : }
6733 :
6734 : unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6735 928 : if (RetVT.SimpleTy != MVT::i64)
6736 : return 0;
6737 928 : return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6738 : }
6739 :
6740 6 : unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6741 6 : if (RetVT.SimpleTy != MVT::v8i16)
6742 : return 0;
6743 6 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6744 2 : return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6745 : }
6746 8 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6747 2 : return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6748 : }
6749 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
6750 2 : return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6751 : }
6752 : return 0;
6753 : }
6754 :
6755 2 : unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6756 2 : if (RetVT.SimpleTy != MVT::v16i16)
6757 : return 0;
6758 2 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6759 0 : return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6760 : }
6761 4 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
6762 2 : return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6763 : }
6764 : return 0;
6765 : }
6766 :
6767 : unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6768 0 : if (RetVT.SimpleTy != MVT::v32i16)
6769 : return 0;
6770 0 : if ((Subtarget->hasBWI())) {
6771 0 : return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6772 : }
6773 : return 0;
6774 : }
6775 :
6776 6 : unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6777 6 : if (RetVT.SimpleTy != MVT::v4i32)
6778 : return 0;
6779 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6780 2 : return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6781 : }
6782 4 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
6783 2 : return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6784 : }
6785 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6786 2 : return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6787 : }
6788 : return 0;
6789 : }
6790 :
6791 2 : unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6792 2 : if (RetVT.SimpleTy != MVT::v8i32)
6793 : return 0;
6794 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6795 0 : return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6796 : }
6797 2 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6798 2 : return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6799 : }
6800 : return 0;
6801 : }
6802 :
6803 : unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6804 0 : if (RetVT.SimpleTy != MVT::v16i32)
6805 : return 0;
6806 0 : if ((Subtarget->hasAVX512())) {
6807 0 : return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6808 : }
6809 : return 0;
6810 : }
6811 :
6812 12 : unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6813 12 : if (RetVT.SimpleTy != MVT::v2i64)
6814 : return 0;
6815 12 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6816 4 : return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6817 : }
6818 : return 0;
6819 : }
6820 :
6821 4 : unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6822 4 : if (RetVT.SimpleTy != MVT::v4i64)
6823 : return 0;
6824 4 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
6825 0 : return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6826 : }
6827 : return 0;
6828 : }
6829 :
6830 : unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6831 4 : if (RetVT.SimpleTy != MVT::v8i64)
6832 : return 0;
6833 4 : if ((Subtarget->hasDQI())) {
6834 0 : return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6835 : }
6836 : return 0;
6837 : }
6838 :
6839 1323 : unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6840 1323 : switch (VT.SimpleTy) {
6841 2 : case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6842 0 : case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6843 357 : case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6844 928 : case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6845 6 : case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6846 2 : case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6847 0 : case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6848 6 : case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6849 2 : case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6850 0 : case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6851 12 : case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6852 4 : case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6853 4 : case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6854 : default: return 0;
6855 : }
6856 : }
6857 :
6858 : // FastEmit functions for ISD::MULHS.
6859 :
6860 0 : unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6861 0 : if (RetVT.SimpleTy != MVT::v8i16)
6862 : return 0;
6863 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6864 0 : return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6865 : }
6866 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6867 0 : return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6868 : }
6869 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
6870 0 : return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6871 : }
6872 : return 0;
6873 : }
6874 :
6875 0 : unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6876 0 : if (RetVT.SimpleTy != MVT::v16i16)
6877 : return 0;
6878 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6879 0 : return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6880 : }
6881 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
6882 0 : return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6883 : }
6884 : return 0;
6885 : }
6886 :
6887 : unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6888 0 : if (RetVT.SimpleTy != MVT::v32i16)
6889 : return 0;
6890 0 : if ((Subtarget->hasBWI())) {
6891 0 : return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6892 : }
6893 : return 0;
6894 : }
6895 :
6896 0 : unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6897 0 : switch (VT.SimpleTy) {
6898 0 : case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6899 0 : case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6900 0 : case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6901 : default: return 0;
6902 : }
6903 : }
6904 :
6905 : // FastEmit functions for ISD::MULHU.
6906 :
6907 0 : unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6908 0 : if (RetVT.SimpleTy != MVT::v8i16)
6909 : return 0;
6910 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6911 0 : return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6912 : }
6913 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6914 0 : return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6915 : }
6916 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
6917 0 : return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6918 : }
6919 : return 0;
6920 : }
6921 :
6922 0 : unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6923 0 : if (RetVT.SimpleTy != MVT::v16i16)
6924 : return 0;
6925 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6926 0 : return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6927 : }
6928 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
6929 0 : return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6930 : }
6931 : return 0;
6932 : }
6933 :
6934 : unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6935 0 : if (RetVT.SimpleTy != MVT::v32i16)
6936 : return 0;
6937 0 : if ((Subtarget->hasBWI())) {
6938 0 : return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6939 : }
6940 : return 0;
6941 : }
6942 :
6943 0 : unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6944 0 : switch (VT.SimpleTy) {
6945 0 : case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6946 0 : case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6947 0 : case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
6948 : default: return 0;
6949 : }
6950 : }
6951 :
6952 : // FastEmit functions for ISD::OR.
6953 :
6954 : unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6955 99 : if (RetVT.SimpleTy != MVT::i8)
6956 : return 0;
6957 99 : return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6958 : }
6959 :
6960 : unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6961 2 : if (RetVT.SimpleTy != MVT::i16)
6962 : return 0;
6963 2 : return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6964 : }
6965 :
6966 : unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6967 5880 : if (RetVT.SimpleTy != MVT::i32)
6968 : return 0;
6969 5880 : return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6970 : }
6971 :
6972 : unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6973 1983 : if (RetVT.SimpleTy != MVT::i64)
6974 : return 0;
6975 1983 : return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6976 : }
6977 :
6978 : unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6979 0 : if (RetVT.SimpleTy != MVT::v8i1)
6980 : return 0;
6981 0 : if ((Subtarget->hasDQI())) {
6982 0 : return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6983 : }
6984 : return 0;
6985 : }
6986 :
6987 : unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6988 0 : if (RetVT.SimpleTy != MVT::v16i1)
6989 : return 0;
6990 0 : if ((Subtarget->hasAVX512())) {
6991 0 : return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
6992 : }
6993 : return 0;
6994 : }
6995 :
6996 : unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
6997 0 : if (RetVT.SimpleTy != MVT::v32i1)
6998 : return 0;
6999 0 : if ((Subtarget->hasBWI())) {
7000 0 : return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7001 : }
7002 : return 0;
7003 : }
7004 :
7005 : unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7006 0 : if (RetVT.SimpleTy != MVT::v64i1)
7007 : return 0;
7008 0 : if ((Subtarget->hasBWI())) {
7009 0 : return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7010 : }
7011 : return 0;
7012 : }
7013 :
7014 8 : unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7015 8 : if (RetVT.SimpleTy != MVT::v2i64)
7016 : return 0;
7017 16 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7018 2 : return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7019 : }
7020 6 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7021 2 : return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7022 : }
7023 4 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7024 4 : return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7025 : }
7026 : return 0;
7027 : }
7028 :
7029 6 : unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7030 6 : if (RetVT.SimpleTy != MVT::v4i64)
7031 : return 0;
7032 12 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
7033 4 : return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7034 : }
7035 2 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7036 0 : return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7037 : }
7038 2 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7039 2 : return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7040 : }
7041 : return 0;
7042 : }
7043 :
7044 : unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7045 0 : if (RetVT.SimpleTy != MVT::v8i64)
7046 : return 0;
7047 0 : if ((Subtarget->hasAVX512())) {
7048 0 : return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7049 : }
7050 : return 0;
7051 : }
7052 :
7053 7990 : unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7054 7990 : switch (VT.SimpleTy) {
7055 99 : case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7056 2 : case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7057 5880 : case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7058 1983 : case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7059 0 : case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7060 0 : case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7061 0 : case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7062 0 : case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7063 8 : case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7064 6 : case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7065 0 : case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7066 : default: return 0;
7067 : }
7068 : }
7069 :
7070 : // FastEmit functions for ISD::ROTL.
7071 :
7072 0 : unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7073 0 : if (RetVT.SimpleTy != MVT::i8)
7074 : return 0;
7075 0 : BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
7076 0 : return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
7077 : }
7078 :
7079 : unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7080 0 : if (RetVT.SimpleTy != MVT::v16i8)
7081 : return 0;
7082 0 : if ((Subtarget->hasXOP())) {
7083 0 : return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7084 : }
7085 : return 0;
7086 : }
7087 :
7088 : unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7089 0 : if (RetVT.SimpleTy != MVT::v8i16)
7090 : return 0;
7091 0 : if ((Subtarget->hasXOP())) {
7092 0 : return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7093 : }
7094 : return 0;
7095 : }
7096 :
7097 0 : unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7098 0 : if (RetVT.SimpleTy != MVT::v4i32)
7099 : return 0;
7100 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7101 0 : return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7102 : }
7103 0 : if ((Subtarget->hasXOP())) {
7104 0 : return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7105 : }
7106 : return 0;
7107 : }
7108 :
7109 0 : unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7110 0 : if (RetVT.SimpleTy != MVT::v8i32)
7111 : return 0;
7112 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7113 0 : return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7114 : }
7115 : return 0;
7116 : }
7117 :
7118 : unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7119 0 : if (RetVT.SimpleTy != MVT::v16i32)
7120 : return 0;
7121 0 : if ((Subtarget->hasAVX512())) {
7122 0 : return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7123 : }
7124 : return 0;
7125 : }
7126 :
7127 0 : unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7128 0 : if (RetVT.SimpleTy != MVT::v2i64)
7129 : return 0;
7130 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7131 0 : return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7132 : }
7133 0 : if ((Subtarget->hasXOP())) {
7134 0 : return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7135 : }
7136 : return 0;
7137 : }
7138 :
7139 0 : unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7140 0 : if (RetVT.SimpleTy != MVT::v4i64)
7141 : return 0;
7142 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7143 0 : return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7144 : }
7145 : return 0;
7146 : }
7147 :
7148 : unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7149 0 : if (RetVT.SimpleTy != MVT::v8i64)
7150 : return 0;
7151 0 : if ((Subtarget->hasAVX512())) {
7152 0 : return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7153 : }
7154 : return 0;
7155 : }
7156 :
7157 0 : unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7158 0 : switch (VT.SimpleTy) {
7159 0 : case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7160 0 : case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7161 0 : case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7162 0 : case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7163 0 : case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7164 0 : case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7165 0 : case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7166 0 : case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7167 0 : case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7168 : default: return 0;
7169 : }
7170 : }
7171 :
7172 : // FastEmit functions for ISD::ROTR.
7173 :
7174 0 : unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7175 0 : if (RetVT.SimpleTy != MVT::i8)
7176 : return 0;
7177 0 : BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
7178 0 : return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
7179 : }
7180 :
7181 0 : unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7182 0 : if (RetVT.SimpleTy != MVT::v4i32)
7183 : return 0;
7184 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7185 0 : return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7186 : }
7187 : return 0;
7188 : }
7189 :
7190 0 : unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7191 0 : if (RetVT.SimpleTy != MVT::v8i32)
7192 : return 0;
7193 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7194 0 : return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7195 : }
7196 : return 0;
7197 : }
7198 :
7199 : unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7200 0 : if (RetVT.SimpleTy != MVT::v16i32)
7201 : return 0;
7202 0 : if ((Subtarget->hasAVX512())) {
7203 0 : return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7204 : }
7205 : return 0;
7206 : }
7207 :
7208 0 : unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7209 0 : if (RetVT.SimpleTy != MVT::v2i64)
7210 : return 0;
7211 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7212 0 : return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7213 : }
7214 : return 0;
7215 : }
7216 :
7217 0 : unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7218 0 : if (RetVT.SimpleTy != MVT::v4i64)
7219 : return 0;
7220 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7221 0 : return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7222 : }
7223 : return 0;
7224 : }
7225 :
7226 : unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7227 0 : if (RetVT.SimpleTy != MVT::v8i64)
7228 : return 0;
7229 0 : if ((Subtarget->hasAVX512())) {
7230 0 : return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7231 : }
7232 : return 0;
7233 : }
7234 :
7235 0 : unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7236 0 : switch (VT.SimpleTy) {
7237 0 : case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7238 0 : case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7239 0 : case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7240 0 : case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7241 0 : case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7242 0 : case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7243 0 : case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7244 : default: return 0;
7245 : }
7246 : }
7247 :
7248 : // FastEmit functions for ISD::SHL.
7249 :
7250 3 : unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7251 3 : if (RetVT.SimpleTy != MVT::i8)
7252 : return 0;
7253 6 : BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
7254 3 : return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
7255 : }
7256 :
7257 0 : unsigned fastEmit_ISD_SHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7258 0 : if (RetVT.SimpleTy != MVT::v8i16)
7259 : return 0;
7260 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7261 0 : return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7262 : }
7263 : return 0;
7264 : }
7265 :
7266 0 : unsigned fastEmit_ISD_SHL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7267 0 : if (RetVT.SimpleTy != MVT::v16i16)
7268 : return 0;
7269 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7270 0 : return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7271 : }
7272 : return 0;
7273 : }
7274 :
7275 : unsigned fastEmit_ISD_SHL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7276 0 : if (RetVT.SimpleTy != MVT::v32i16)
7277 : return 0;
7278 0 : if ((Subtarget->hasBWI())) {
7279 0 : return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7280 : }
7281 : return 0;
7282 : }
7283 :
7284 0 : unsigned fastEmit_ISD_SHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7285 0 : if (RetVT.SimpleTy != MVT::v4i32)
7286 : return 0;
7287 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7288 0 : return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7289 : }
7290 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7291 0 : return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7292 : }
7293 : return 0;
7294 : }
7295 :
7296 0 : unsigned fastEmit_ISD_SHL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7297 0 : if (RetVT.SimpleTy != MVT::v8i32)
7298 : return 0;
7299 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7300 0 : return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7301 : }
7302 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7303 0 : return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7304 : }
7305 : return 0;
7306 : }
7307 :
7308 : unsigned fastEmit_ISD_SHL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7309 0 : if (RetVT.SimpleTy != MVT::v16i32)
7310 : return 0;
7311 0 : if ((Subtarget->hasAVX512())) {
7312 0 : return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7313 : }
7314 : return 0;
7315 : }
7316 :
7317 0 : unsigned fastEmit_ISD_SHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7318 0 : if (RetVT.SimpleTy != MVT::v2i64)
7319 : return 0;
7320 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7321 0 : return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7322 : }
7323 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7324 0 : return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7325 : }
7326 : return 0;
7327 : }
7328 :
7329 0 : unsigned fastEmit_ISD_SHL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7330 0 : if (RetVT.SimpleTy != MVT::v4i64)
7331 : return 0;
7332 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7333 0 : return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7334 : }
7335 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7336 0 : return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7337 : }
7338 : return 0;
7339 : }
7340 :
7341 : unsigned fastEmit_ISD_SHL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7342 0 : if (RetVT.SimpleTy != MVT::v8i64)
7343 : return 0;
7344 0 : if ((Subtarget->hasAVX512())) {
7345 0 : return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7346 : }
7347 : return 0;
7348 : }
7349 :
7350 3169 : unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7351 3169 : switch (VT.SimpleTy) {
7352 3 : case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7353 0 : case MVT::v8i16: return fastEmit_ISD_SHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7354 0 : case MVT::v16i16: return fastEmit_ISD_SHL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7355 0 : case MVT::v32i16: return fastEmit_ISD_SHL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7356 0 : case MVT::v4i32: return fastEmit_ISD_SHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7357 0 : case MVT::v8i32: return fastEmit_ISD_SHL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7358 0 : case MVT::v16i32: return fastEmit_ISD_SHL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7359 0 : case MVT::v2i64: return fastEmit_ISD_SHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7360 0 : case MVT::v4i64: return fastEmit_ISD_SHL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7361 0 : case MVT::v8i64: return fastEmit_ISD_SHL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7362 : default: return 0;
7363 : }
7364 : }
7365 :
7366 : // FastEmit functions for ISD::SMAX.
7367 :
7368 0 : unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7369 0 : if (RetVT.SimpleTy != MVT::v16i8)
7370 : return 0;
7371 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7372 0 : return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7373 : }
7374 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7375 0 : return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7376 : }
7377 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7378 0 : return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7379 : }
7380 : return 0;
7381 : }
7382 :
7383 0 : unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7384 0 : if (RetVT.SimpleTy != MVT::v32i8)
7385 : return 0;
7386 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7387 0 : return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7388 : }
7389 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7390 0 : return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7391 : }
7392 : return 0;
7393 : }
7394 :
7395 : unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7396 0 : if (RetVT.SimpleTy != MVT::v64i8)
7397 : return 0;
7398 0 : if ((Subtarget->hasBWI())) {
7399 0 : return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7400 : }
7401 : return 0;
7402 : }
7403 :
7404 0 : unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7405 0 : if (RetVT.SimpleTy != MVT::v8i16)
7406 : return 0;
7407 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7408 0 : return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7409 : }
7410 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7411 0 : return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7412 : }
7413 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7414 0 : return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7415 : }
7416 : return 0;
7417 : }
7418 :
7419 0 : unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7420 0 : if (RetVT.SimpleTy != MVT::v16i16)
7421 : return 0;
7422 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7423 0 : return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7424 : }
7425 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7426 0 : return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7427 : }
7428 : return 0;
7429 : }
7430 :
7431 : unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7432 0 : if (RetVT.SimpleTy != MVT::v32i16)
7433 : return 0;
7434 0 : if ((Subtarget->hasBWI())) {
7435 0 : return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7436 : }
7437 : return 0;
7438 : }
7439 :
7440 0 : unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7441 0 : if (RetVT.SimpleTy != MVT::v4i32)
7442 : return 0;
7443 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7444 0 : return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7445 : }
7446 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7447 0 : return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7448 : }
7449 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7450 0 : return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7451 : }
7452 : return 0;
7453 : }
7454 :
7455 0 : unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7456 0 : if (RetVT.SimpleTy != MVT::v8i32)
7457 : return 0;
7458 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7459 0 : return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7460 : }
7461 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7462 0 : return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7463 : }
7464 : return 0;
7465 : }
7466 :
7467 : unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7468 0 : if (RetVT.SimpleTy != MVT::v16i32)
7469 : return 0;
7470 0 : if ((Subtarget->hasAVX512())) {
7471 0 : return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7472 : }
7473 : return 0;
7474 : }
7475 :
7476 0 : unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7477 0 : if (RetVT.SimpleTy != MVT::v2i64)
7478 : return 0;
7479 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7480 0 : return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7481 : }
7482 : return 0;
7483 : }
7484 :
7485 0 : unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7486 0 : if (RetVT.SimpleTy != MVT::v4i64)
7487 : return 0;
7488 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7489 0 : return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7490 : }
7491 : return 0;
7492 : }
7493 :
7494 : unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7495 0 : if (RetVT.SimpleTy != MVT::v8i64)
7496 : return 0;
7497 0 : if ((Subtarget->hasAVX512())) {
7498 0 : return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7499 : }
7500 : return 0;
7501 : }
7502 :
7503 0 : unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7504 0 : switch (VT.SimpleTy) {
7505 0 : case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7506 0 : case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7507 0 : case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7508 0 : case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7509 0 : case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7510 0 : case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7511 0 : case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7512 0 : case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7513 0 : case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7514 0 : case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7515 0 : case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7516 0 : case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7517 : default: return 0;
7518 : }
7519 : }
7520 :
7521 : // FastEmit functions for ISD::SMIN.
7522 :
7523 0 : unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7524 0 : if (RetVT.SimpleTy != MVT::v16i8)
7525 : return 0;
7526 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7527 0 : return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7528 : }
7529 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7530 0 : return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7531 : }
7532 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7533 0 : return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7534 : }
7535 : return 0;
7536 : }
7537 :
7538 0 : unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7539 0 : if (RetVT.SimpleTy != MVT::v32i8)
7540 : return 0;
7541 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7542 0 : return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7543 : }
7544 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7545 0 : return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7546 : }
7547 : return 0;
7548 : }
7549 :
7550 : unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7551 0 : if (RetVT.SimpleTy != MVT::v64i8)
7552 : return 0;
7553 0 : if ((Subtarget->hasBWI())) {
7554 0 : return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7555 : }
7556 : return 0;
7557 : }
7558 :
7559 0 : unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7560 0 : if (RetVT.SimpleTy != MVT::v8i16)
7561 : return 0;
7562 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7563 0 : return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7564 : }
7565 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7566 0 : return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7567 : }
7568 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7569 0 : return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7570 : }
7571 : return 0;
7572 : }
7573 :
7574 0 : unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7575 0 : if (RetVT.SimpleTy != MVT::v16i16)
7576 : return 0;
7577 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7578 0 : return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7579 : }
7580 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7581 0 : return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7582 : }
7583 : return 0;
7584 : }
7585 :
7586 : unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7587 0 : if (RetVT.SimpleTy != MVT::v32i16)
7588 : return 0;
7589 0 : if ((Subtarget->hasBWI())) {
7590 0 : return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7591 : }
7592 : return 0;
7593 : }
7594 :
7595 0 : unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7596 0 : if (RetVT.SimpleTy != MVT::v4i32)
7597 : return 0;
7598 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7599 0 : return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7600 : }
7601 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
7602 0 : return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7603 : }
7604 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7605 0 : return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7606 : }
7607 : return 0;
7608 : }
7609 :
7610 0 : unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7611 0 : if (RetVT.SimpleTy != MVT::v8i32)
7612 : return 0;
7613 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7614 0 : return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7615 : }
7616 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7617 0 : return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7618 : }
7619 : return 0;
7620 : }
7621 :
7622 : unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7623 0 : if (RetVT.SimpleTy != MVT::v16i32)
7624 : return 0;
7625 0 : if ((Subtarget->hasAVX512())) {
7626 0 : return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7627 : }
7628 : return 0;
7629 : }
7630 :
7631 0 : unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7632 0 : if (RetVT.SimpleTy != MVT::v2i64)
7633 : return 0;
7634 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7635 0 : return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7636 : }
7637 : return 0;
7638 : }
7639 :
7640 0 : unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7641 0 : if (RetVT.SimpleTy != MVT::v4i64)
7642 : return 0;
7643 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7644 0 : return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7645 : }
7646 : return 0;
7647 : }
7648 :
7649 : unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7650 0 : if (RetVT.SimpleTy != MVT::v8i64)
7651 : return 0;
7652 0 : if ((Subtarget->hasAVX512())) {
7653 0 : return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7654 : }
7655 : return 0;
7656 : }
7657 :
7658 0 : unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7659 0 : switch (VT.SimpleTy) {
7660 0 : case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7661 0 : case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7662 0 : case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7663 0 : case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7664 0 : case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7665 0 : case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7666 0 : case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7667 0 : case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7668 0 : case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7669 0 : case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7670 0 : case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7671 0 : case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7672 : default: return 0;
7673 : }
7674 : }
7675 :
7676 : // FastEmit functions for ISD::SRA.
7677 :
7678 1 : unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7679 1 : if (RetVT.SimpleTy != MVT::i8)
7680 : return 0;
7681 2 : BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
7682 1 : return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
7683 : }
7684 :
7685 0 : unsigned fastEmit_ISD_SRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7686 0 : if (RetVT.SimpleTy != MVT::v8i16)
7687 : return 0;
7688 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7689 0 : return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7690 : }
7691 : return 0;
7692 : }
7693 :
7694 0 : unsigned fastEmit_ISD_SRA_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7695 0 : if (RetVT.SimpleTy != MVT::v16i16)
7696 : return 0;
7697 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7698 0 : return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7699 : }
7700 : return 0;
7701 : }
7702 :
7703 : unsigned fastEmit_ISD_SRA_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7704 0 : if (RetVT.SimpleTy != MVT::v32i16)
7705 : return 0;
7706 0 : if ((Subtarget->hasBWI())) {
7707 0 : return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7708 : }
7709 : return 0;
7710 : }
7711 :
7712 0 : unsigned fastEmit_ISD_SRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7713 0 : if (RetVT.SimpleTy != MVT::v4i32)
7714 : return 0;
7715 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7716 0 : return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7717 : }
7718 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7719 0 : return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7720 : }
7721 : return 0;
7722 : }
7723 :
7724 0 : unsigned fastEmit_ISD_SRA_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7725 0 : if (RetVT.SimpleTy != MVT::v8i32)
7726 : return 0;
7727 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7728 0 : return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7729 : }
7730 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7731 0 : return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7732 : }
7733 : return 0;
7734 : }
7735 :
7736 : unsigned fastEmit_ISD_SRA_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7737 0 : if (RetVT.SimpleTy != MVT::v16i32)
7738 : return 0;
7739 0 : if ((Subtarget->hasAVX512())) {
7740 0 : return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7741 : }
7742 : return 0;
7743 : }
7744 :
7745 0 : unsigned fastEmit_ISD_SRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7746 0 : if (RetVT.SimpleTy != MVT::v2i64)
7747 : return 0;
7748 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7749 0 : return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7750 : }
7751 : return 0;
7752 : }
7753 :
7754 0 : unsigned fastEmit_ISD_SRA_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7755 0 : if (RetVT.SimpleTy != MVT::v4i64)
7756 : return 0;
7757 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7758 0 : return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7759 : }
7760 : return 0;
7761 : }
7762 :
7763 : unsigned fastEmit_ISD_SRA_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7764 0 : if (RetVT.SimpleTy != MVT::v8i64)
7765 : return 0;
7766 0 : if ((Subtarget->hasAVX512())) {
7767 0 : return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7768 : }
7769 : return 0;
7770 : }
7771 :
7772 18 : unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7773 18 : switch (VT.SimpleTy) {
7774 1 : case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7775 0 : case MVT::v8i16: return fastEmit_ISD_SRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7776 0 : case MVT::v16i16: return fastEmit_ISD_SRA_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7777 0 : case MVT::v32i16: return fastEmit_ISD_SRA_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7778 0 : case MVT::v4i32: return fastEmit_ISD_SRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7779 0 : case MVT::v8i32: return fastEmit_ISD_SRA_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7780 0 : case MVT::v16i32: return fastEmit_ISD_SRA_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7781 0 : case MVT::v2i64: return fastEmit_ISD_SRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7782 0 : case MVT::v4i64: return fastEmit_ISD_SRA_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7783 0 : case MVT::v8i64: return fastEmit_ISD_SRA_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7784 : default: return 0;
7785 : }
7786 : }
7787 :
7788 : // FastEmit functions for ISD::SRL.
7789 :
7790 1 : unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7791 1 : if (RetVT.SimpleTy != MVT::i8)
7792 : return 0;
7793 2 : BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
7794 1 : return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
7795 : }
7796 :
7797 0 : unsigned fastEmit_ISD_SRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7798 0 : if (RetVT.SimpleTy != MVT::v8i16)
7799 : return 0;
7800 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7801 0 : return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7802 : }
7803 : return 0;
7804 : }
7805 :
7806 0 : unsigned fastEmit_ISD_SRL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7807 0 : if (RetVT.SimpleTy != MVT::v16i16)
7808 : return 0;
7809 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7810 0 : return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7811 : }
7812 : return 0;
7813 : }
7814 :
7815 : unsigned fastEmit_ISD_SRL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7816 0 : if (RetVT.SimpleTy != MVT::v32i16)
7817 : return 0;
7818 0 : if ((Subtarget->hasBWI())) {
7819 0 : return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7820 : }
7821 : return 0;
7822 : }
7823 :
7824 0 : unsigned fastEmit_ISD_SRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7825 0 : if (RetVT.SimpleTy != MVT::v4i32)
7826 : return 0;
7827 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7828 0 : return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7829 : }
7830 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7831 0 : return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7832 : }
7833 : return 0;
7834 : }
7835 :
7836 0 : unsigned fastEmit_ISD_SRL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7837 0 : if (RetVT.SimpleTy != MVT::v8i32)
7838 : return 0;
7839 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7840 0 : return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7841 : }
7842 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7843 0 : return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7844 : }
7845 : return 0;
7846 : }
7847 :
7848 : unsigned fastEmit_ISD_SRL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7849 0 : if (RetVT.SimpleTy != MVT::v16i32)
7850 : return 0;
7851 0 : if ((Subtarget->hasAVX512())) {
7852 0 : return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7853 : }
7854 : return 0;
7855 : }
7856 :
7857 0 : unsigned fastEmit_ISD_SRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7858 0 : if (RetVT.SimpleTy != MVT::v2i64)
7859 : return 0;
7860 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7861 0 : return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7862 : }
7863 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7864 0 : return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7865 : }
7866 : return 0;
7867 : }
7868 :
7869 0 : unsigned fastEmit_ISD_SRL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7870 0 : if (RetVT.SimpleTy != MVT::v4i64)
7871 : return 0;
7872 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7873 0 : return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7874 : }
7875 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7876 0 : return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7877 : }
7878 : return 0;
7879 : }
7880 :
7881 : unsigned fastEmit_ISD_SRL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7882 0 : if (RetVT.SimpleTy != MVT::v8i64)
7883 : return 0;
7884 0 : if ((Subtarget->hasAVX512())) {
7885 0 : return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7886 : }
7887 : return 0;
7888 : }
7889 :
7890 3607 : unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7891 3607 : switch (VT.SimpleTy) {
7892 1 : case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7893 0 : case MVT::v8i16: return fastEmit_ISD_SRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7894 0 : case MVT::v16i16: return fastEmit_ISD_SRL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7895 0 : case MVT::v32i16: return fastEmit_ISD_SRL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7896 0 : case MVT::v4i32: return fastEmit_ISD_SRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7897 0 : case MVT::v8i32: return fastEmit_ISD_SRL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7898 0 : case MVT::v16i32: return fastEmit_ISD_SRL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7899 0 : case MVT::v2i64: return fastEmit_ISD_SRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7900 0 : case MVT::v4i64: return fastEmit_ISD_SRL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7901 0 : case MVT::v8i64: return fastEmit_ISD_SRL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7902 : default: return 0;
7903 : }
7904 : }
7905 :
7906 : // FastEmit functions for ISD::SUB.
7907 :
7908 : unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7909 0 : if (RetVT.SimpleTy != MVT::i8)
7910 : return 0;
7911 0 : return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7912 : }
7913 :
7914 : unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7915 0 : if (RetVT.SimpleTy != MVT::i16)
7916 : return 0;
7917 0 : return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7918 : }
7919 :
7920 : unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7921 4765 : if (RetVT.SimpleTy != MVT::i32)
7922 : return 0;
7923 4765 : return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7924 : }
7925 :
7926 : unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7927 71561 : if (RetVT.SimpleTy != MVT::i64)
7928 : return 0;
7929 71561 : return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7930 : }
7931 :
7932 12 : unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7933 12 : if (RetVT.SimpleTy != MVT::v16i8)
7934 : return 0;
7935 12 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7936 4 : return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7937 : }
7938 16 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7939 4 : return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7940 : }
7941 4 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7942 4 : return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7943 : }
7944 : return 0;
7945 : }
7946 :
7947 4 : unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7948 4 : if (RetVT.SimpleTy != MVT::v32i8)
7949 : return 0;
7950 4 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7951 0 : return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7952 : }
7953 8 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7954 4 : return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7955 : }
7956 : return 0;
7957 : }
7958 :
7959 : unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7960 0 : if (RetVT.SimpleTy != MVT::v64i8)
7961 : return 0;
7962 0 : if ((Subtarget->hasBWI())) {
7963 0 : return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7964 : }
7965 : return 0;
7966 : }
7967 :
7968 12 : unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7969 12 : if (RetVT.SimpleTy != MVT::v8i16)
7970 : return 0;
7971 12 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7972 4 : return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7973 : }
7974 16 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7975 4 : return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7976 : }
7977 4 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7978 4 : return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7979 : }
7980 : return 0;
7981 : }
7982 :
7983 4 : unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7984 4 : if (RetVT.SimpleTy != MVT::v16i16)
7985 : return 0;
7986 4 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7987 0 : return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7988 : }
7989 8 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7990 4 : return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7991 : }
7992 : return 0;
7993 : }
7994 :
7995 : unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7996 0 : if (RetVT.SimpleTy != MVT::v32i16)
7997 : return 0;
7998 0 : if ((Subtarget->hasBWI())) {
7999 0 : return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8000 : }
8001 : return 0;
8002 : }
8003 :
8004 6 : unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8005 6 : if (RetVT.SimpleTy != MVT::v4i32)
8006 : return 0;
8007 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8008 2 : return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8009 : }
8010 4 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8011 2 : return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8012 : }
8013 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8014 2 : return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8015 : }
8016 : return 0;
8017 : }
8018 :
8019 2 : unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8020 2 : if (RetVT.SimpleTy != MVT::v8i32)
8021 : return 0;
8022 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8023 0 : return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8024 : }
8025 2 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8026 2 : return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8027 : }
8028 : return 0;
8029 : }
8030 :
8031 : unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8032 0 : if (RetVT.SimpleTy != MVT::v16i32)
8033 : return 0;
8034 0 : if ((Subtarget->hasAVX512())) {
8035 0 : return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8036 : }
8037 : return 0;
8038 : }
8039 :
8040 6 : unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8041 6 : if (RetVT.SimpleTy != MVT::v2i64)
8042 : return 0;
8043 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8044 2 : return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8045 : }
8046 4 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8047 2 : return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8048 : }
8049 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8050 2 : return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8051 : }
8052 : return 0;
8053 : }
8054 :
8055 2 : unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8056 2 : if (RetVT.SimpleTy != MVT::v4i64)
8057 : return 0;
8058 4 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8059 0 : return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8060 : }
8061 2 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8062 2 : return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8063 : }
8064 : return 0;
8065 : }
8066 :
8067 : unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8068 0 : if (RetVT.SimpleTy != MVT::v8i64)
8069 : return 0;
8070 0 : if ((Subtarget->hasAVX512())) {
8071 0 : return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8072 : }
8073 : return 0;
8074 : }
8075 :
8076 76374 : unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8077 76374 : switch (VT.SimpleTy) {
8078 0 : case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8079 0 : case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8080 4765 : case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8081 71561 : case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8082 12 : case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8083 4 : case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8084 0 : case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8085 12 : case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8086 4 : case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8087 0 : case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8088 6 : case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8089 2 : case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8090 0 : case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8091 6 : case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8092 2 : case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8093 0 : case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8094 : default: return 0;
8095 : }
8096 : }
8097 :
8098 : // FastEmit functions for ISD::UMAX.
8099 :
8100 0 : unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8101 0 : if (RetVT.SimpleTy != MVT::v16i8)
8102 : return 0;
8103 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8104 0 : return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8105 : }
8106 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8107 0 : return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8108 : }
8109 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8110 0 : return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8111 : }
8112 : return 0;
8113 : }
8114 :
8115 0 : unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8116 0 : if (RetVT.SimpleTy != MVT::v32i8)
8117 : return 0;
8118 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8119 0 : return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8120 : }
8121 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8122 0 : return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8123 : }
8124 : return 0;
8125 : }
8126 :
8127 : unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8128 0 : if (RetVT.SimpleTy != MVT::v64i8)
8129 : return 0;
8130 0 : if ((Subtarget->hasBWI())) {
8131 0 : return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8132 : }
8133 : return 0;
8134 : }
8135 :
8136 0 : unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8137 0 : if (RetVT.SimpleTy != MVT::v8i16)
8138 : return 0;
8139 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8140 0 : return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8141 : }
8142 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8143 0 : return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8144 : }
8145 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8146 0 : return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8147 : }
8148 : return 0;
8149 : }
8150 :
8151 0 : unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8152 0 : if (RetVT.SimpleTy != MVT::v16i16)
8153 : return 0;
8154 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8155 0 : return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8156 : }
8157 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8158 0 : return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8159 : }
8160 : return 0;
8161 : }
8162 :
8163 : unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8164 0 : if (RetVT.SimpleTy != MVT::v32i16)
8165 : return 0;
8166 0 : if ((Subtarget->hasBWI())) {
8167 0 : return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8168 : }
8169 : return 0;
8170 : }
8171 :
8172 0 : unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8173 0 : if (RetVT.SimpleTy != MVT::v4i32)
8174 : return 0;
8175 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8176 0 : return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8177 : }
8178 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8179 0 : return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8180 : }
8181 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8182 0 : return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8183 : }
8184 : return 0;
8185 : }
8186 :
8187 0 : unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8188 0 : if (RetVT.SimpleTy != MVT::v8i32)
8189 : return 0;
8190 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8191 0 : return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8192 : }
8193 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8194 0 : return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8195 : }
8196 : return 0;
8197 : }
8198 :
8199 : unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8200 0 : if (RetVT.SimpleTy != MVT::v16i32)
8201 : return 0;
8202 0 : if ((Subtarget->hasAVX512())) {
8203 0 : return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8204 : }
8205 : return 0;
8206 : }
8207 :
8208 0 : unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8209 0 : if (RetVT.SimpleTy != MVT::v2i64)
8210 : return 0;
8211 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8212 0 : return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8213 : }
8214 : return 0;
8215 : }
8216 :
8217 0 : unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8218 0 : if (RetVT.SimpleTy != MVT::v4i64)
8219 : return 0;
8220 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8221 0 : return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8222 : }
8223 : return 0;
8224 : }
8225 :
8226 : unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8227 0 : if (RetVT.SimpleTy != MVT::v8i64)
8228 : return 0;
8229 0 : if ((Subtarget->hasAVX512())) {
8230 0 : return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8231 : }
8232 : return 0;
8233 : }
8234 :
8235 0 : unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8236 0 : switch (VT.SimpleTy) {
8237 0 : case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8238 0 : case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8239 0 : case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8240 0 : case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8241 0 : case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8242 0 : case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8243 0 : case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8244 0 : case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8245 0 : case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8246 0 : case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8247 0 : case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8248 0 : case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8249 : default: return 0;
8250 : }
8251 : }
8252 :
8253 : // FastEmit functions for ISD::UMIN.
8254 :
8255 0 : unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8256 0 : if (RetVT.SimpleTy != MVT::v16i8)
8257 : return 0;
8258 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8259 0 : return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8260 : }
8261 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8262 0 : return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8263 : }
8264 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8265 0 : return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8266 : }
8267 : return 0;
8268 : }
8269 :
8270 0 : unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8271 0 : if (RetVT.SimpleTy != MVT::v32i8)
8272 : return 0;
8273 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8274 0 : return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8275 : }
8276 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8277 0 : return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8278 : }
8279 : return 0;
8280 : }
8281 :
8282 : unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8283 0 : if (RetVT.SimpleTy != MVT::v64i8)
8284 : return 0;
8285 0 : if ((Subtarget->hasBWI())) {
8286 0 : return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8287 : }
8288 : return 0;
8289 : }
8290 :
8291 0 : unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8292 0 : if (RetVT.SimpleTy != MVT::v8i16)
8293 : return 0;
8294 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8295 0 : return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8296 : }
8297 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8298 0 : return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8299 : }
8300 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8301 0 : return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8302 : }
8303 : return 0;
8304 : }
8305 :
8306 0 : unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8307 0 : if (RetVT.SimpleTy != MVT::v16i16)
8308 : return 0;
8309 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8310 0 : return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8311 : }
8312 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8313 0 : return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8314 : }
8315 : return 0;
8316 : }
8317 :
8318 : unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8319 0 : if (RetVT.SimpleTy != MVT::v32i16)
8320 : return 0;
8321 0 : if ((Subtarget->hasBWI())) {
8322 0 : return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8323 : }
8324 : return 0;
8325 : }
8326 :
8327 0 : unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8328 0 : if (RetVT.SimpleTy != MVT::v4i32)
8329 : return 0;
8330 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8331 0 : return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8332 : }
8333 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8334 0 : return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8335 : }
8336 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8337 0 : return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8338 : }
8339 : return 0;
8340 : }
8341 :
8342 0 : unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8343 0 : if (RetVT.SimpleTy != MVT::v8i32)
8344 : return 0;
8345 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8346 0 : return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8347 : }
8348 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8349 0 : return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8350 : }
8351 : return 0;
8352 : }
8353 :
8354 : unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8355 0 : if (RetVT.SimpleTy != MVT::v16i32)
8356 : return 0;
8357 0 : if ((Subtarget->hasAVX512())) {
8358 0 : return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8359 : }
8360 : return 0;
8361 : }
8362 :
8363 0 : unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8364 0 : if (RetVT.SimpleTy != MVT::v2i64)
8365 : return 0;
8366 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8367 0 : return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8368 : }
8369 : return 0;
8370 : }
8371 :
8372 0 : unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8373 0 : if (RetVT.SimpleTy != MVT::v4i64)
8374 : return 0;
8375 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8376 0 : return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8377 : }
8378 : return 0;
8379 : }
8380 :
8381 : unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8382 0 : if (RetVT.SimpleTy != MVT::v8i64)
8383 : return 0;
8384 0 : if ((Subtarget->hasAVX512())) {
8385 0 : return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8386 : }
8387 : return 0;
8388 : }
8389 :
8390 0 : unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8391 0 : switch (VT.SimpleTy) {
8392 0 : case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8393 0 : case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8394 0 : case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8395 0 : case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8396 0 : case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8397 0 : case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8398 0 : case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8399 0 : case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8400 0 : case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8401 0 : case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8402 0 : case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8403 0 : case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8404 : default: return 0;
8405 : }
8406 : }
8407 :
8408 : // FastEmit functions for ISD::XOR.
8409 :
8410 : unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8411 0 : if (RetVT.SimpleTy != MVT::i8)
8412 : return 0;
8413 0 : return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8414 : }
8415 :
8416 : unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8417 0 : if (RetVT.SimpleTy != MVT::i16)
8418 : return 0;
8419 0 : return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8420 : }
8421 :
8422 : unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8423 267 : if (RetVT.SimpleTy != MVT::i32)
8424 : return 0;
8425 267 : return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8426 : }
8427 :
8428 : unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8429 1304 : if (RetVT.SimpleTy != MVT::i64)
8430 : return 0;
8431 1304 : return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8432 : }
8433 :
8434 : unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8435 0 : if (RetVT.SimpleTy != MVT::v8i1)
8436 : return 0;
8437 0 : if ((Subtarget->hasDQI())) {
8438 0 : return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8439 : }
8440 : return 0;
8441 : }
8442 :
8443 : unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8444 0 : if (RetVT.SimpleTy != MVT::v16i1)
8445 : return 0;
8446 0 : if ((Subtarget->hasAVX512())) {
8447 0 : return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8448 : }
8449 : return 0;
8450 : }
8451 :
8452 : unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8453 0 : if (RetVT.SimpleTy != MVT::v32i1)
8454 : return 0;
8455 0 : if ((Subtarget->hasBWI())) {
8456 0 : return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8457 : }
8458 : return 0;
8459 : }
8460 :
8461 : unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8462 0 : if (RetVT.SimpleTy != MVT::v64i1)
8463 : return 0;
8464 0 : if ((Subtarget->hasBWI())) {
8465 0 : return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8466 : }
8467 : return 0;
8468 : }
8469 :
8470 6 : unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8471 6 : if (RetVT.SimpleTy != MVT::v2i64)
8472 : return 0;
8473 12 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8474 2 : return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8475 : }
8476 4 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8477 2 : return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8478 : }
8479 2 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8480 2 : return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8481 : }
8482 : return 0;
8483 : }
8484 :
8485 4 : unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8486 4 : if (RetVT.SimpleTy != MVT::v4i64)
8487 : return 0;
8488 8 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
8489 2 : return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8490 : }
8491 2 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8492 0 : return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8493 : }
8494 2 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8495 2 : return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8496 : }
8497 : return 0;
8498 : }
8499 :
8500 : unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8501 0 : if (RetVT.SimpleTy != MVT::v8i64)
8502 : return 0;
8503 0 : if ((Subtarget->hasAVX512())) {
8504 0 : return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8505 : }
8506 : return 0;
8507 : }
8508 :
8509 1593 : unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8510 1593 : switch (VT.SimpleTy) {
8511 0 : case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8512 0 : case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8513 267 : case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8514 1304 : case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8515 0 : case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8516 0 : case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8517 0 : case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8518 0 : case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8519 6 : case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8520 4 : case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8521 0 : case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8522 : default: return 0;
8523 : }
8524 : }
8525 :
8526 : // FastEmit functions for X86ISD::ADDS.
8527 :
8528 0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8529 0 : if (RetVT.SimpleTy != MVT::v16i8)
8530 : return 0;
8531 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8532 0 : return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8533 : }
8534 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8535 0 : return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8536 : }
8537 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8538 0 : return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8539 : }
8540 : return 0;
8541 : }
8542 :
8543 0 : unsigned fastEmit_X86ISD_ADDS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8544 0 : if (RetVT.SimpleTy != MVT::v32i8)
8545 : return 0;
8546 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8547 0 : return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8548 : }
8549 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8550 0 : return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8551 : }
8552 : return 0;
8553 : }
8554 :
8555 : unsigned fastEmit_X86ISD_ADDS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8556 0 : if (RetVT.SimpleTy != MVT::v64i8)
8557 : return 0;
8558 0 : if ((Subtarget->hasBWI())) {
8559 0 : return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8560 : }
8561 : return 0;
8562 : }
8563 :
8564 0 : unsigned fastEmit_X86ISD_ADDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8565 0 : if (RetVT.SimpleTy != MVT::v8i16)
8566 : return 0;
8567 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8568 0 : return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8569 : }
8570 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8571 0 : return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8572 : }
8573 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8574 0 : return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8575 : }
8576 : return 0;
8577 : }
8578 :
8579 0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8580 0 : if (RetVT.SimpleTy != MVT::v16i16)
8581 : return 0;
8582 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8583 0 : return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8584 : }
8585 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8586 0 : return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8587 : }
8588 : return 0;
8589 : }
8590 :
8591 : unsigned fastEmit_X86ISD_ADDS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8592 0 : if (RetVT.SimpleTy != MVT::v32i16)
8593 : return 0;
8594 0 : if ((Subtarget->hasBWI())) {
8595 0 : return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8596 : }
8597 : return 0;
8598 : }
8599 :
8600 0 : unsigned fastEmit_X86ISD_ADDS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8601 0 : switch (VT.SimpleTy) {
8602 0 : case MVT::v16i8: return fastEmit_X86ISD_ADDS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8603 0 : case MVT::v32i8: return fastEmit_X86ISD_ADDS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8604 0 : case MVT::v64i8: return fastEmit_X86ISD_ADDS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8605 0 : case MVT::v8i16: return fastEmit_X86ISD_ADDS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8606 0 : case MVT::v16i16: return fastEmit_X86ISD_ADDS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8607 0 : case MVT::v32i16: return fastEmit_X86ISD_ADDS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8608 : default: return 0;
8609 : }
8610 : }
8611 :
8612 : // FastEmit functions for X86ISD::ADDSUB.
8613 :
8614 0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8615 0 : if (RetVT.SimpleTy != MVT::v4f32)
8616 : return 0;
8617 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
8618 0 : return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8619 : }
8620 0 : if ((Subtarget->hasAVX())) {
8621 0 : return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8622 : }
8623 : return 0;
8624 : }
8625 :
8626 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8627 0 : if (RetVT.SimpleTy != MVT::v8f32)
8628 : return 0;
8629 0 : if ((Subtarget->hasAVX())) {
8630 0 : return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8631 : }
8632 : return 0;
8633 : }
8634 :
8635 0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8636 0 : if (RetVT.SimpleTy != MVT::v2f64)
8637 : return 0;
8638 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
8639 0 : return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8640 : }
8641 0 : if ((Subtarget->hasAVX())) {
8642 0 : return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8643 : }
8644 : return 0;
8645 : }
8646 :
8647 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8648 0 : if (RetVT.SimpleTy != MVT::v4f64)
8649 : return 0;
8650 0 : if ((Subtarget->hasAVX())) {
8651 0 : return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8652 : }
8653 : return 0;
8654 : }
8655 :
8656 0 : unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8657 0 : switch (VT.SimpleTy) {
8658 0 : case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8659 0 : case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8660 0 : case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8661 0 : case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8662 : default: return 0;
8663 : }
8664 : }
8665 :
8666 : // FastEmit functions for X86ISD::ADDUS.
8667 :
8668 0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8669 0 : if (RetVT.SimpleTy != MVT::v16i8)
8670 : return 0;
8671 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8672 0 : return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8673 : }
8674 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8675 0 : return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8676 : }
8677 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8678 0 : return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8679 : }
8680 : return 0;
8681 : }
8682 :
8683 0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8684 0 : if (RetVT.SimpleTy != MVT::v32i8)
8685 : return 0;
8686 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8687 0 : return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8688 : }
8689 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8690 0 : return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8691 : }
8692 : return 0;
8693 : }
8694 :
8695 : unsigned fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8696 0 : if (RetVT.SimpleTy != MVT::v64i8)
8697 : return 0;
8698 0 : if ((Subtarget->hasBWI())) {
8699 0 : return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8700 : }
8701 : return 0;
8702 : }
8703 :
8704 0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8705 0 : if (RetVT.SimpleTy != MVT::v8i16)
8706 : return 0;
8707 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8708 0 : return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8709 : }
8710 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8711 0 : return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8712 : }
8713 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8714 0 : return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8715 : }
8716 : return 0;
8717 : }
8718 :
8719 0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8720 0 : if (RetVT.SimpleTy != MVT::v16i16)
8721 : return 0;
8722 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8723 0 : return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8724 : }
8725 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8726 0 : return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8727 : }
8728 : return 0;
8729 : }
8730 :
8731 : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8732 0 : if (RetVT.SimpleTy != MVT::v32i16)
8733 : return 0;
8734 0 : if ((Subtarget->hasBWI())) {
8735 0 : return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8736 : }
8737 : return 0;
8738 : }
8739 :
8740 0 : unsigned fastEmit_X86ISD_ADDUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8741 0 : switch (VT.SimpleTy) {
8742 0 : case MVT::v16i8: return fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8743 0 : case MVT::v32i8: return fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8744 0 : case MVT::v64i8: return fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8745 0 : case MVT::v8i16: return fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8746 0 : case MVT::v16i16: return fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8747 0 : case MVT::v32i16: return fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8748 : default: return 0;
8749 : }
8750 : }
8751 :
8752 : // FastEmit functions for X86ISD::ANDNP.
8753 :
8754 0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8755 0 : if (RetVT.SimpleTy != MVT::v2i64)
8756 : return 0;
8757 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8758 0 : return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8759 : }
8760 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8761 0 : return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8762 : }
8763 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8764 0 : return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8765 : }
8766 : return 0;
8767 : }
8768 :
8769 0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8770 0 : if (RetVT.SimpleTy != MVT::v4i64)
8771 : return 0;
8772 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
8773 0 : return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8774 : }
8775 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8776 0 : return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8777 : }
8778 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8779 0 : return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8780 : }
8781 : return 0;
8782 : }
8783 :
8784 : unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8785 0 : if (RetVT.SimpleTy != MVT::v8i64)
8786 : return 0;
8787 0 : if ((Subtarget->hasAVX512())) {
8788 0 : return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8789 : }
8790 : return 0;
8791 : }
8792 :
8793 0 : unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8794 0 : switch (VT.SimpleTy) {
8795 0 : case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8796 0 : case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8797 0 : case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8798 : default: return 0;
8799 : }
8800 : }
8801 :
8802 : // FastEmit functions for X86ISD::AVG.
8803 :
8804 0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8805 0 : if (RetVT.SimpleTy != MVT::v16i8)
8806 : return 0;
8807 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8808 0 : return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8809 : }
8810 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8811 0 : return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8812 : }
8813 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8814 0 : return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8815 : }
8816 : return 0;
8817 : }
8818 :
8819 0 : unsigned fastEmit_X86ISD_AVG_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8820 0 : if (RetVT.SimpleTy != MVT::v32i8)
8821 : return 0;
8822 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8823 0 : return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8824 : }
8825 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8826 0 : return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8827 : }
8828 : return 0;
8829 : }
8830 :
8831 : unsigned fastEmit_X86ISD_AVG_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8832 0 : if (RetVT.SimpleTy != MVT::v64i8)
8833 : return 0;
8834 0 : if ((Subtarget->hasBWI())) {
8835 0 : return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8836 : }
8837 : return 0;
8838 : }
8839 :
8840 0 : unsigned fastEmit_X86ISD_AVG_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8841 0 : if (RetVT.SimpleTy != MVT::v8i16)
8842 : return 0;
8843 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8844 0 : return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8845 : }
8846 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8847 0 : return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8848 : }
8849 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8850 0 : return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8851 : }
8852 : return 0;
8853 : }
8854 :
8855 0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8856 0 : if (RetVT.SimpleTy != MVT::v16i16)
8857 : return 0;
8858 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8859 0 : return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8860 : }
8861 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8862 0 : return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8863 : }
8864 : return 0;
8865 : }
8866 :
8867 : unsigned fastEmit_X86ISD_AVG_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8868 0 : if (RetVT.SimpleTy != MVT::v32i16)
8869 : return 0;
8870 0 : if ((Subtarget->hasBWI())) {
8871 0 : return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8872 : }
8873 : return 0;
8874 : }
8875 :
8876 0 : unsigned fastEmit_X86ISD_AVG_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8877 0 : switch (VT.SimpleTy) {
8878 0 : case MVT::v16i8: return fastEmit_X86ISD_AVG_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8879 0 : case MVT::v32i8: return fastEmit_X86ISD_AVG_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8880 0 : case MVT::v64i8: return fastEmit_X86ISD_AVG_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8881 0 : case MVT::v8i16: return fastEmit_X86ISD_AVG_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8882 0 : case MVT::v16i16: return fastEmit_X86ISD_AVG_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8883 0 : case MVT::v32i16: return fastEmit_X86ISD_AVG_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8884 : default: return 0;
8885 : }
8886 : }
8887 :
8888 : // FastEmit functions for X86ISD::BEXTR.
8889 :
8890 : unsigned fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8891 0 : if (RetVT.SimpleTy != MVT::i32)
8892 : return 0;
8893 0 : if ((Subtarget->hasBMI())) {
8894 0 : return fastEmitInst_rr(X86::BEXTR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8895 : }
8896 : return 0;
8897 : }
8898 :
8899 : unsigned fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8900 0 : if (RetVT.SimpleTy != MVT::i64)
8901 : return 0;
8902 0 : if ((Subtarget->hasBMI())) {
8903 0 : return fastEmitInst_rr(X86::BEXTR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8904 : }
8905 : return 0;
8906 : }
8907 :
8908 0 : unsigned fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8909 0 : switch (VT.SimpleTy) {
8910 0 : case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8911 0 : case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8912 : default: return 0;
8913 : }
8914 : }
8915 :
8916 : // FastEmit functions for X86ISD::BT.
8917 :
8918 : unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8919 0 : if (RetVT.SimpleTy != MVT::i32)
8920 : return 0;
8921 0 : return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8922 : }
8923 :
8924 : unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8925 0 : if (RetVT.SimpleTy != MVT::i32)
8926 : return 0;
8927 0 : return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8928 : }
8929 :
8930 : unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8931 0 : if (RetVT.SimpleTy != MVT::i32)
8932 : return 0;
8933 0 : return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8934 : }
8935 :
8936 0 : unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8937 0 : switch (VT.SimpleTy) {
8938 0 : case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8939 0 : case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8940 0 : case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8941 : default: return 0;
8942 : }
8943 : }
8944 :
8945 : // FastEmit functions for X86ISD::CMP.
8946 :
8947 : unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8948 0 : if (RetVT.SimpleTy != MVT::i32)
8949 : return 0;
8950 0 : return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8951 : }
8952 :
8953 : unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8954 0 : if (RetVT.SimpleTy != MVT::i32)
8955 : return 0;
8956 0 : return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8957 : }
8958 :
8959 : unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8960 0 : if (RetVT.SimpleTy != MVT::i32)
8961 : return 0;
8962 0 : return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8963 : }
8964 :
8965 : unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8966 0 : if (RetVT.SimpleTy != MVT::i32)
8967 : return 0;
8968 0 : return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8969 : }
8970 :
8971 0 : unsigned fastEmit_X86ISD_CMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8972 0 : if (RetVT.SimpleTy != MVT::i32)
8973 : return 0;
8974 0 : if ((Subtarget->hasAVX512())) {
8975 0 : return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8976 : }
8977 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
8978 0 : return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8979 : }
8980 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
8981 0 : return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8982 : }
8983 0 : if ((!Subtarget->hasSSE1()) && (Subtarget->hasCMov())) {
8984 0 : return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8985 : }
8986 : return 0;
8987 : }
8988 :
8989 0 : unsigned fastEmit_X86ISD_CMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8990 0 : if (RetVT.SimpleTy != MVT::i32)
8991 : return 0;
8992 0 : if ((Subtarget->hasAVX512())) {
8993 0 : return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8994 : }
8995 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8996 0 : return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8997 : }
8998 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
8999 0 : return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9000 : }
9001 0 : if ((!Subtarget->hasSSE2()) && (Subtarget->hasCMov())) {
9002 0 : return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9003 : }
9004 : return 0;
9005 : }
9006 :
9007 0 : unsigned fastEmit_X86ISD_CMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9008 0 : if (RetVT.SimpleTy != MVT::i32)
9009 : return 0;
9010 0 : if ((Subtarget->hasCMov())) {
9011 0 : return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9012 : }
9013 : return 0;
9014 : }
9015 :
9016 0 : unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9017 0 : switch (VT.SimpleTy) {
9018 0 : case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9019 0 : case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9020 0 : case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9021 0 : case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9022 0 : case MVT::f32: return fastEmit_X86ISD_CMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9023 0 : case MVT::f64: return fastEmit_X86ISD_CMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9024 0 : case MVT::f80: return fastEmit_X86ISD_CMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9025 : default: return 0;
9026 : }
9027 : }
9028 :
9029 : // FastEmit functions for X86ISD::COMI.
9030 :
9031 0 : unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9032 0 : if (RetVT.SimpleTy != MVT::i32)
9033 : return 0;
9034 0 : if ((Subtarget->hasAVX512())) {
9035 0 : return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9036 : }
9037 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9038 0 : return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9039 : }
9040 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9041 0 : return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9042 : }
9043 : return 0;
9044 : }
9045 :
9046 0 : unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9047 0 : if (RetVT.SimpleTy != MVT::i32)
9048 : return 0;
9049 0 : if ((Subtarget->hasAVX512())) {
9050 0 : return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9051 : }
9052 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9053 0 : return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9054 : }
9055 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9056 0 : return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9057 : }
9058 : return 0;
9059 : }
9060 :
9061 0 : unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9062 0 : switch (VT.SimpleTy) {
9063 0 : case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9064 0 : case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9065 : default: return 0;
9066 : }
9067 : }
9068 :
9069 : // FastEmit functions for X86ISD::FAND.
9070 :
9071 0 : unsigned fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9072 0 : if (RetVT.SimpleTy != MVT::f128)
9073 : return 0;
9074 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9075 0 : return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9076 : }
9077 : return 0;
9078 : }
9079 :
9080 : unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9081 0 : if (RetVT.SimpleTy != MVT::v4f32)
9082 : return 0;
9083 0 : return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9084 : }
9085 :
9086 0 : unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9087 0 : switch (VT.SimpleTy) {
9088 0 : case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9089 0 : case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9090 : default: return 0;
9091 : }
9092 : }
9093 :
9094 : // FastEmit functions for X86ISD::FANDN.
9095 :
9096 : unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9097 0 : if (RetVT.SimpleTy != MVT::v4f32)
9098 : return 0;
9099 0 : return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9100 : }
9101 :
9102 : unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9103 0 : switch (VT.SimpleTy) {
9104 0 : case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9105 : default: return 0;
9106 : }
9107 : }
9108 :
9109 : // FastEmit functions for X86ISD::FHADD.
9110 :
9111 0 : unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9112 0 : if (RetVT.SimpleTy != MVT::v4f32)
9113 : return 0;
9114 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
9115 0 : return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9116 : }
9117 0 : if ((Subtarget->hasAVX())) {
9118 0 : return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9119 : }
9120 : return 0;
9121 : }
9122 :
9123 : unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9124 0 : if (RetVT.SimpleTy != MVT::v8f32)
9125 : return 0;
9126 0 : if ((Subtarget->hasAVX())) {
9127 0 : return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9128 : }
9129 : return 0;
9130 : }
9131 :
9132 0 : unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9133 0 : if (RetVT.SimpleTy != MVT::v2f64)
9134 : return 0;
9135 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
9136 0 : return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9137 : }
9138 0 : if ((Subtarget->hasAVX())) {
9139 0 : return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9140 : }
9141 : return 0;
9142 : }
9143 :
9144 : unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9145 0 : if (RetVT.SimpleTy != MVT::v4f64)
9146 : return 0;
9147 0 : if ((Subtarget->hasAVX())) {
9148 0 : return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9149 : }
9150 : return 0;
9151 : }
9152 :
9153 0 : unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9154 0 : switch (VT.SimpleTy) {
9155 0 : case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9156 0 : case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9157 0 : case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9158 0 : case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9159 : default: return 0;
9160 : }
9161 : }
9162 :
9163 : // FastEmit functions for X86ISD::FHSUB.
9164 :
9165 0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9166 0 : if (RetVT.SimpleTy != MVT::v4f32)
9167 : return 0;
9168 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
9169 0 : return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9170 : }
9171 0 : if ((Subtarget->hasAVX())) {
9172 0 : return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9173 : }
9174 : return 0;
9175 : }
9176 :
9177 : unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9178 0 : if (RetVT.SimpleTy != MVT::v8f32)
9179 : return 0;
9180 0 : if ((Subtarget->hasAVX())) {
9181 0 : return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9182 : }
9183 : return 0;
9184 : }
9185 :
9186 0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9187 0 : if (RetVT.SimpleTy != MVT::v2f64)
9188 : return 0;
9189 0 : if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
9190 0 : return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9191 : }
9192 0 : if ((Subtarget->hasAVX())) {
9193 0 : return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9194 : }
9195 : return 0;
9196 : }
9197 :
9198 : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9199 0 : if (RetVT.SimpleTy != MVT::v4f64)
9200 : return 0;
9201 0 : if ((Subtarget->hasAVX())) {
9202 0 : return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9203 : }
9204 : return 0;
9205 : }
9206 :
9207 0 : unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9208 0 : switch (VT.SimpleTy) {
9209 0 : case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9210 0 : case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9211 0 : case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9212 0 : case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9213 : default: return 0;
9214 : }
9215 : }
9216 :
9217 : // FastEmit functions for X86ISD::FMAX.
9218 :
9219 0 : unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9220 0 : if (RetVT.SimpleTy != MVT::f32)
9221 : return 0;
9222 0 : if ((Subtarget->hasAVX512())) {
9223 0 : return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9224 : }
9225 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9226 0 : return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9227 : }
9228 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9229 0 : return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9230 : }
9231 : return 0;
9232 : }
9233 :
9234 0 : unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9235 0 : if (RetVT.SimpleTy != MVT::f64)
9236 : return 0;
9237 0 : if ((Subtarget->hasAVX512())) {
9238 0 : return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9239 : }
9240 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9241 0 : return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9242 : }
9243 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9244 0 : return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9245 : }
9246 : return 0;
9247 : }
9248 :
9249 0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9250 0 : if (RetVT.SimpleTy != MVT::v4f32)
9251 : return 0;
9252 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9253 0 : return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9254 : }
9255 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9256 0 : return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9257 : }
9258 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9259 0 : return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9260 : }
9261 : return 0;
9262 : }
9263 :
9264 0 : unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9265 0 : if (RetVT.SimpleTy != MVT::v8f32)
9266 : return 0;
9267 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9268 0 : return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9269 : }
9270 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9271 0 : return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9272 : }
9273 : return 0;
9274 : }
9275 :
9276 : unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9277 0 : if (RetVT.SimpleTy != MVT::v16f32)
9278 : return 0;
9279 0 : if ((Subtarget->hasAVX512())) {
9280 0 : return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9281 : }
9282 : return 0;
9283 : }
9284 :
9285 0 : unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9286 0 : if (RetVT.SimpleTy != MVT::v2f64)
9287 : return 0;
9288 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9289 0 : return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9290 : }
9291 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9292 0 : return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9293 : }
9294 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9295 0 : return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9296 : }
9297 : return 0;
9298 : }
9299 :
9300 0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9301 0 : if (RetVT.SimpleTy != MVT::v4f64)
9302 : return 0;
9303 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9304 0 : return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9305 : }
9306 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9307 0 : return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9308 : }
9309 : return 0;
9310 : }
9311 :
9312 : unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9313 0 : if (RetVT.SimpleTy != MVT::v8f64)
9314 : return 0;
9315 0 : if ((Subtarget->hasAVX512())) {
9316 0 : return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9317 : }
9318 : return 0;
9319 : }
9320 :
9321 0 : unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9322 0 : switch (VT.SimpleTy) {
9323 0 : case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9324 0 : case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9325 0 : case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9326 0 : case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9327 0 : case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9328 0 : case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9329 0 : case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9330 0 : case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9331 : default: return 0;
9332 : }
9333 : }
9334 :
9335 : // FastEmit functions for X86ISD::FMAXC.
9336 :
9337 0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9338 0 : if (RetVT.SimpleTy != MVT::f32)
9339 : return 0;
9340 0 : if ((Subtarget->hasAVX512())) {
9341 0 : return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9342 : }
9343 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9344 0 : return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9345 : }
9346 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9347 0 : return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9348 : }
9349 : return 0;
9350 : }
9351 :
9352 0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9353 0 : if (RetVT.SimpleTy != MVT::f64)
9354 : return 0;
9355 0 : if ((Subtarget->hasAVX512())) {
9356 0 : return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9357 : }
9358 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9359 0 : return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9360 : }
9361 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9362 0 : return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9363 : }
9364 : return 0;
9365 : }
9366 :
9367 0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9368 0 : if (RetVT.SimpleTy != MVT::v4f32)
9369 : return 0;
9370 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9371 0 : return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9372 : }
9373 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9374 0 : return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9375 : }
9376 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9377 0 : return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9378 : }
9379 : return 0;
9380 : }
9381 :
9382 0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9383 0 : if (RetVT.SimpleTy != MVT::v8f32)
9384 : return 0;
9385 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9386 0 : return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9387 : }
9388 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9389 0 : return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9390 : }
9391 : return 0;
9392 : }
9393 :
9394 : unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9395 0 : if (RetVT.SimpleTy != MVT::v16f32)
9396 : return 0;
9397 0 : if ((Subtarget->hasAVX512())) {
9398 0 : return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9399 : }
9400 : return 0;
9401 : }
9402 :
9403 0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9404 0 : if (RetVT.SimpleTy != MVT::v2f64)
9405 : return 0;
9406 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9407 0 : return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9408 : }
9409 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9410 0 : return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9411 : }
9412 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9413 0 : return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9414 : }
9415 : return 0;
9416 : }
9417 :
9418 0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9419 0 : if (RetVT.SimpleTy != MVT::v4f64)
9420 : return 0;
9421 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9422 0 : return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9423 : }
9424 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9425 0 : return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9426 : }
9427 : return 0;
9428 : }
9429 :
9430 : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9431 0 : if (RetVT.SimpleTy != MVT::v8f64)
9432 : return 0;
9433 0 : if ((Subtarget->hasAVX512())) {
9434 0 : return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9435 : }
9436 : return 0;
9437 : }
9438 :
9439 0 : unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9440 0 : switch (VT.SimpleTy) {
9441 0 : case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9442 0 : case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9443 0 : case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9444 0 : case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9445 0 : case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9446 0 : case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9447 0 : case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9448 0 : case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9449 : default: return 0;
9450 : }
9451 : }
9452 :
9453 : // FastEmit functions for X86ISD::FMAXS.
9454 :
9455 0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9456 0 : if (RetVT.SimpleTy != MVT::v4f32)
9457 : return 0;
9458 0 : if ((Subtarget->hasAVX512())) {
9459 0 : return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9460 : }
9461 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9462 0 : return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9463 : }
9464 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9465 0 : return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9466 : }
9467 : return 0;
9468 : }
9469 :
9470 0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9471 0 : if (RetVT.SimpleTy != MVT::v2f64)
9472 : return 0;
9473 0 : if ((Subtarget->hasAVX512())) {
9474 0 : return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9475 : }
9476 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9477 0 : return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9478 : }
9479 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9480 0 : return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9481 : }
9482 : return 0;
9483 : }
9484 :
9485 0 : unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9486 0 : switch (VT.SimpleTy) {
9487 0 : case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9488 0 : case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9489 : default: return 0;
9490 : }
9491 : }
9492 :
9493 : // FastEmit functions for X86ISD::FMIN.
9494 :
9495 0 : unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9496 0 : if (RetVT.SimpleTy != MVT::f32)
9497 : return 0;
9498 0 : if ((Subtarget->hasAVX512())) {
9499 0 : return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9500 : }
9501 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9502 0 : return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9503 : }
9504 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9505 0 : return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9506 : }
9507 : return 0;
9508 : }
9509 :
9510 0 : unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9511 0 : if (RetVT.SimpleTy != MVT::f64)
9512 : return 0;
9513 0 : if ((Subtarget->hasAVX512())) {
9514 0 : return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9515 : }
9516 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9517 0 : return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9518 : }
9519 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9520 0 : return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9521 : }
9522 : return 0;
9523 : }
9524 :
9525 0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9526 0 : if (RetVT.SimpleTy != MVT::v4f32)
9527 : return 0;
9528 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9529 0 : return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9530 : }
9531 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9532 0 : return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9533 : }
9534 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9535 0 : return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9536 : }
9537 : return 0;
9538 : }
9539 :
9540 0 : unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9541 0 : if (RetVT.SimpleTy != MVT::v8f32)
9542 : return 0;
9543 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9544 0 : return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9545 : }
9546 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9547 0 : return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9548 : }
9549 : return 0;
9550 : }
9551 :
9552 : unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9553 0 : if (RetVT.SimpleTy != MVT::v16f32)
9554 : return 0;
9555 0 : if ((Subtarget->hasAVX512())) {
9556 0 : return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9557 : }
9558 : return 0;
9559 : }
9560 :
9561 0 : unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9562 0 : if (RetVT.SimpleTy != MVT::v2f64)
9563 : return 0;
9564 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9565 0 : return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9566 : }
9567 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9568 0 : return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9569 : }
9570 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9571 0 : return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9572 : }
9573 : return 0;
9574 : }
9575 :
9576 0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9577 0 : if (RetVT.SimpleTy != MVT::v4f64)
9578 : return 0;
9579 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9580 0 : return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9581 : }
9582 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9583 0 : return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9584 : }
9585 : return 0;
9586 : }
9587 :
9588 : unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9589 0 : if (RetVT.SimpleTy != MVT::v8f64)
9590 : return 0;
9591 0 : if ((Subtarget->hasAVX512())) {
9592 0 : return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9593 : }
9594 : return 0;
9595 : }
9596 :
9597 0 : unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9598 0 : switch (VT.SimpleTy) {
9599 0 : case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9600 0 : case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9601 0 : case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9602 0 : case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9603 0 : case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9604 0 : case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9605 0 : case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9606 0 : case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9607 : default: return 0;
9608 : }
9609 : }
9610 :
9611 : // FastEmit functions for X86ISD::FMINC.
9612 :
9613 0 : unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9614 0 : if (RetVT.SimpleTy != MVT::f32)
9615 : return 0;
9616 0 : if ((Subtarget->hasAVX512())) {
9617 0 : return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9618 : }
9619 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9620 0 : return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9621 : }
9622 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9623 0 : return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9624 : }
9625 : return 0;
9626 : }
9627 :
9628 0 : unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9629 0 : if (RetVT.SimpleTy != MVT::f64)
9630 : return 0;
9631 0 : if ((Subtarget->hasAVX512())) {
9632 0 : return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9633 : }
9634 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9635 0 : return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9636 : }
9637 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9638 0 : return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9639 : }
9640 : return 0;
9641 : }
9642 :
9643 0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9644 0 : if (RetVT.SimpleTy != MVT::v4f32)
9645 : return 0;
9646 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9647 0 : return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9648 : }
9649 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9650 0 : return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9651 : }
9652 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9653 0 : return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9654 : }
9655 : return 0;
9656 : }
9657 :
9658 0 : unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9659 0 : if (RetVT.SimpleTy != MVT::v8f32)
9660 : return 0;
9661 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9662 0 : return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9663 : }
9664 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9665 0 : return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9666 : }
9667 : return 0;
9668 : }
9669 :
9670 : unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9671 0 : if (RetVT.SimpleTy != MVT::v16f32)
9672 : return 0;
9673 0 : if ((Subtarget->hasAVX512())) {
9674 0 : return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9675 : }
9676 : return 0;
9677 : }
9678 :
9679 0 : unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9680 0 : if (RetVT.SimpleTy != MVT::v2f64)
9681 : return 0;
9682 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9683 0 : return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9684 : }
9685 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9686 0 : return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9687 : }
9688 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9689 0 : return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9690 : }
9691 : return 0;
9692 : }
9693 :
9694 0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9695 0 : if (RetVT.SimpleTy != MVT::v4f64)
9696 : return 0;
9697 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9698 0 : return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9699 : }
9700 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9701 0 : return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9702 : }
9703 : return 0;
9704 : }
9705 :
9706 : unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9707 0 : if (RetVT.SimpleTy != MVT::v8f64)
9708 : return 0;
9709 0 : if ((Subtarget->hasAVX512())) {
9710 0 : return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9711 : }
9712 : return 0;
9713 : }
9714 :
9715 0 : unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9716 0 : switch (VT.SimpleTy) {
9717 0 : case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9718 0 : case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9719 0 : case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9720 0 : case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9721 0 : case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9722 0 : case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9723 0 : case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9724 0 : case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9725 : default: return 0;
9726 : }
9727 : }
9728 :
9729 : // FastEmit functions for X86ISD::FMINS.
9730 :
9731 0 : unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9732 0 : if (RetVT.SimpleTy != MVT::v4f32)
9733 : return 0;
9734 0 : if ((Subtarget->hasAVX512())) {
9735 0 : return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9736 : }
9737 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9738 0 : return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9739 : }
9740 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9741 0 : return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9742 : }
9743 : return 0;
9744 : }
9745 :
9746 0 : unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9747 0 : if (RetVT.SimpleTy != MVT::v2f64)
9748 : return 0;
9749 0 : if ((Subtarget->hasAVX512())) {
9750 0 : return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9751 : }
9752 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9753 0 : return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9754 : }
9755 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9756 0 : return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9757 : }
9758 : return 0;
9759 : }
9760 :
9761 0 : unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9762 0 : switch (VT.SimpleTy) {
9763 0 : case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9764 0 : case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9765 : default: return 0;
9766 : }
9767 : }
9768 :
9769 : // FastEmit functions for X86ISD::FOR.
9770 :
9771 0 : unsigned fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9772 0 : if (RetVT.SimpleTy != MVT::f128)
9773 : return 0;
9774 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9775 0 : return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9776 : }
9777 : return 0;
9778 : }
9779 :
9780 : unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9781 0 : if (RetVT.SimpleTy != MVT::v4f32)
9782 : return 0;
9783 0 : return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9784 : }
9785 :
9786 0 : unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9787 0 : switch (VT.SimpleTy) {
9788 0 : case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9789 0 : case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9790 : default: return 0;
9791 : }
9792 : }
9793 :
9794 : // FastEmit functions for X86ISD::FXOR.
9795 :
9796 0 : unsigned fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9797 0 : if (RetVT.SimpleTy != MVT::f128)
9798 : return 0;
9799 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9800 0 : return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9801 : }
9802 : return 0;
9803 : }
9804 :
9805 : unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9806 0 : if (RetVT.SimpleTy != MVT::v4f32)
9807 : return 0;
9808 0 : return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9809 : }
9810 :
9811 0 : unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9812 0 : switch (VT.SimpleTy) {
9813 0 : case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9814 0 : case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9815 : default: return 0;
9816 : }
9817 : }
9818 :
9819 : // FastEmit functions for X86ISD::GF2P8MULB.
9820 :
9821 0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9822 0 : if (RetVT.SimpleTy != MVT::v16i8)
9823 : return 0;
9824 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
9825 0 : return fastEmitInst_rr(X86::VGF2P8MULBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9826 : }
9827 0 : if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9828 0 : return fastEmitInst_rr(X86::VGF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9829 : }
9830 0 : if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9831 0 : return fastEmitInst_rr(X86::GF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9832 : }
9833 : return 0;
9834 : }
9835 :
9836 0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9837 0 : if (RetVT.SimpleTy != MVT::v32i8)
9838 : return 0;
9839 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
9840 0 : return fastEmitInst_rr(X86::VGF2P8MULBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9841 : }
9842 0 : if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9843 0 : return fastEmitInst_rr(X86::VGF2P8MULBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9844 : }
9845 : return 0;
9846 : }
9847 :
9848 0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9849 0 : if (RetVT.SimpleTy != MVT::v64i8)
9850 : return 0;
9851 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasBWI()) && (Subtarget->hasGFNI())) {
9852 0 : return fastEmitInst_rr(X86::VGF2P8MULBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9853 : }
9854 : return 0;
9855 : }
9856 :
9857 0 : unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9858 0 : switch (VT.SimpleTy) {
9859 0 : case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9860 0 : case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9861 0 : case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9862 : default: return 0;
9863 : }
9864 : }
9865 :
9866 : // FastEmit functions for X86ISD::HADD.
9867 :
9868 0 : unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9869 0 : if (RetVT.SimpleTy != MVT::v8i16)
9870 : return 0;
9871 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
9872 0 : return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9873 : }
9874 0 : if ((Subtarget->hasAVX())) {
9875 0 : return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9876 : }
9877 : return 0;
9878 : }
9879 :
9880 : unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9881 0 : if (RetVT.SimpleTy != MVT::v16i16)
9882 : return 0;
9883 0 : if ((Subtarget->hasAVX2())) {
9884 0 : return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9885 : }
9886 : return 0;
9887 : }
9888 :
9889 0 : unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9890 0 : if (RetVT.SimpleTy != MVT::v4i32)
9891 : return 0;
9892 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
9893 0 : return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9894 : }
9895 0 : if ((Subtarget->hasAVX())) {
9896 0 : return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9897 : }
9898 : return 0;
9899 : }
9900 :
9901 : unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9902 0 : if (RetVT.SimpleTy != MVT::v8i32)
9903 : return 0;
9904 0 : if ((Subtarget->hasAVX2())) {
9905 0 : return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9906 : }
9907 : return 0;
9908 : }
9909 :
9910 0 : unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9911 0 : switch (VT.SimpleTy) {
9912 0 : case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9913 0 : case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9914 0 : case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9915 0 : case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9916 : default: return 0;
9917 : }
9918 : }
9919 :
9920 : // FastEmit functions for X86ISD::HSUB.
9921 :
9922 0 : unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9923 0 : if (RetVT.SimpleTy != MVT::v8i16)
9924 : return 0;
9925 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
9926 0 : return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9927 : }
9928 0 : if ((Subtarget->hasAVX())) {
9929 0 : return fastEmitInst_rr(X86::VPHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9930 : }
9931 : return 0;
9932 : }
9933 :
9934 : unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9935 0 : if (RetVT.SimpleTy != MVT::v16i16)
9936 : return 0;
9937 0 : if ((Subtarget->hasAVX2())) {
9938 0 : return fastEmitInst_rr(X86::VPHSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9939 : }
9940 : return 0;
9941 : }
9942 :
9943 0 : unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9944 0 : if (RetVT.SimpleTy != MVT::v4i32)
9945 : return 0;
9946 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
9947 0 : return fastEmitInst_rr(X86::PHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9948 : }
9949 0 : if ((Subtarget->hasAVX())) {
9950 0 : return fastEmitInst_rr(X86::VPHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9951 : }
9952 : return 0;
9953 : }
9954 :
9955 : unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9956 0 : if (RetVT.SimpleTy != MVT::v8i32)
9957 : return 0;
9958 0 : if ((Subtarget->hasAVX2())) {
9959 0 : return fastEmitInst_rr(X86::VPHSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9960 : }
9961 : return 0;
9962 : }
9963 :
9964 0 : unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9965 0 : switch (VT.SimpleTy) {
9966 0 : case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9967 0 : case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9968 0 : case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9969 0 : case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9970 : default: return 0;
9971 : }
9972 : }
9973 :
9974 : // FastEmit functions for X86ISD::KADD.
9975 :
9976 : unsigned fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9977 0 : if (RetVT.SimpleTy != MVT::v8i1)
9978 : return 0;
9979 0 : if ((Subtarget->hasDQI())) {
9980 0 : return fastEmitInst_rr(X86::KADDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9981 : }
9982 : return 0;
9983 : }
9984 :
9985 : unsigned fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9986 0 : if (RetVT.SimpleTy != MVT::v16i1)
9987 : return 0;
9988 0 : if ((Subtarget->hasDQI())) {
9989 0 : return fastEmitInst_rr(X86::KADDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9990 : }
9991 : return 0;
9992 : }
9993 :
9994 : unsigned fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9995 0 : if (RetVT.SimpleTy != MVT::v32i1)
9996 : return 0;
9997 0 : if ((Subtarget->hasBWI())) {
9998 0 : return fastEmitInst_rr(X86::KADDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9999 : }
10000 : return 0;
10001 : }
10002 :
10003 : unsigned fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10004 0 : if (RetVT.SimpleTy != MVT::v64i1)
10005 : return 0;
10006 0 : if ((Subtarget->hasBWI())) {
10007 0 : return fastEmitInst_rr(X86::KADDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10008 : }
10009 : return 0;
10010 : }
10011 :
10012 0 : unsigned fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10013 0 : switch (VT.SimpleTy) {
10014 0 : case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10015 0 : case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10016 0 : case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10017 0 : case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10018 : default: return 0;
10019 : }
10020 : }
10021 :
10022 : // FastEmit functions for X86ISD::KORTEST.
10023 :
10024 : unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10025 0 : if (RetVT.SimpleTy != MVT::i32)
10026 : return 0;
10027 0 : if ((Subtarget->hasDQI())) {
10028 0 : return fastEmitInst_rr(X86::KORTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10029 : }
10030 : return 0;
10031 : }
10032 :
10033 : unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10034 0 : if (RetVT.SimpleTy != MVT::i32)
10035 : return 0;
10036 0 : if ((Subtarget->hasAVX512())) {
10037 0 : return fastEmitInst_rr(X86::KORTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10038 : }
10039 : return 0;
10040 : }
10041 :
10042 : unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10043 0 : if (RetVT.SimpleTy != MVT::i32)
10044 : return 0;
10045 0 : if ((Subtarget->hasBWI())) {
10046 0 : return fastEmitInst_rr(X86::KORTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10047 : }
10048 : return 0;
10049 : }
10050 :
10051 : unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10052 0 : if (RetVT.SimpleTy != MVT::i32)
10053 : return 0;
10054 0 : if ((Subtarget->hasBWI())) {
10055 0 : return fastEmitInst_rr(X86::KORTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10056 : }
10057 : return 0;
10058 : }
10059 :
10060 0 : unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10061 0 : switch (VT.SimpleTy) {
10062 0 : case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10063 0 : case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10064 0 : case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10065 0 : case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10066 : default: return 0;
10067 : }
10068 : }
10069 :
10070 : // FastEmit functions for X86ISD::KTEST.
10071 :
10072 : unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10073 0 : if (RetVT.SimpleTy != MVT::i32)
10074 : return 0;
10075 0 : if ((Subtarget->hasDQI())) {
10076 0 : return fastEmitInst_rr(X86::KTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10077 : }
10078 : return 0;
10079 : }
10080 :
10081 : unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10082 0 : if (RetVT.SimpleTy != MVT::i32)
10083 : return 0;
10084 0 : if ((Subtarget->hasDQI())) {
10085 0 : return fastEmitInst_rr(X86::KTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10086 : }
10087 : return 0;
10088 : }
10089 :
10090 : unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10091 0 : if (RetVT.SimpleTy != MVT::i32)
10092 : return 0;
10093 0 : if ((Subtarget->hasBWI())) {
10094 0 : return fastEmitInst_rr(X86::KTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10095 : }
10096 : return 0;
10097 : }
10098 :
10099 : unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10100 0 : if (RetVT.SimpleTy != MVT::i32)
10101 : return 0;
10102 0 : if ((Subtarget->hasBWI())) {
10103 0 : return fastEmitInst_rr(X86::KTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10104 : }
10105 : return 0;
10106 : }
10107 :
10108 0 : unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10109 0 : switch (VT.SimpleTy) {
10110 0 : case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10111 0 : case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10112 0 : case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10113 0 : case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10114 : default: return 0;
10115 : }
10116 : }
10117 :
10118 : // FastEmit functions for X86ISD::MOVHLPS.
10119 :
10120 0 : unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10121 0 : if (RetVT.SimpleTy != MVT::v4f32)
10122 : return 0;
10123 0 : if ((Subtarget->hasAVX512())) {
10124 0 : return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10125 : }
10126 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
10127 0 : return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10128 : }
10129 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10130 0 : return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10131 : }
10132 : return 0;
10133 : }
10134 :
10135 : unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10136 0 : switch (VT.SimpleTy) {
10137 0 : case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10138 : default: return 0;
10139 : }
10140 : }
10141 :
10142 : // FastEmit functions for X86ISD::MOVLHPS.
10143 :
10144 0 : unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10145 0 : if (RetVT.SimpleTy != MVT::v4f32)
10146 : return 0;
10147 0 : if ((Subtarget->hasAVX512())) {
10148 0 : return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10149 : }
10150 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
10151 0 : return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10152 : }
10153 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10154 0 : return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10155 : }
10156 : return 0;
10157 : }
10158 :
10159 : unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10160 0 : switch (VT.SimpleTy) {
10161 0 : case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10162 : default: return 0;
10163 : }
10164 : }
10165 :
10166 : // FastEmit functions for X86ISD::MOVSD.
10167 :
10168 0 : unsigned fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10169 0 : if (RetVT.SimpleTy != MVT::v2f64)
10170 : return 0;
10171 0 : if ((Subtarget->hasAVX512()) && (MF->getFunction().optForSize())) {
10172 0 : return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10173 : }
10174 0 : if ((MF->getFunction().optForSize() || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10175 0 : return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10176 : }
10177 0 : if ((MF->getFunction().optForSize()) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10178 0 : return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10179 : }
10180 : return 0;
10181 : }
10182 :
10183 : unsigned fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10184 0 : switch (VT.SimpleTy) {
10185 0 : case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10186 : default: return 0;
10187 : }
10188 : }
10189 :
10190 : // FastEmit functions for X86ISD::MOVSS.
10191 :
10192 0 : unsigned fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10193 0 : if (RetVT.SimpleTy != MVT::v4f32)
10194 : return 0;
10195 0 : if ((Subtarget->hasAVX512()) && (MF->getFunction().optForSize())) {
10196 0 : return fastEmitInst_rr(X86::VMOVSSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10197 : }
10198 0 : if ((MF->getFunction().optForSize() || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
10199 0 : return fastEmitInst_rr(X86::MOVSSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10200 : }
10201 0 : if ((MF->getFunction().optForSize()) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
10202 0 : return fastEmitInst_rr(X86::VMOVSSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10203 : }
10204 : return 0;
10205 : }
10206 :
10207 : unsigned fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10208 0 : switch (VT.SimpleTy) {
10209 0 : case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10210 : default: return 0;
10211 : }
10212 : }
10213 :
10214 : // FastEmit functions for X86ISD::MULHRS.
10215 :
10216 0 : unsigned fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10217 0 : if (RetVT.SimpleTy != MVT::v8i16)
10218 : return 0;
10219 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10220 0 : return fastEmitInst_rr(X86::VPMULHRSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10221 : }
10222 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
10223 0 : return fastEmitInst_rr(X86::PMULHRSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10224 : }
10225 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10226 0 : return fastEmitInst_rr(X86::VPMULHRSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10227 : }
10228 : return 0;
10229 : }
10230 :
10231 0 : unsigned fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10232 0 : if (RetVT.SimpleTy != MVT::v16i16)
10233 : return 0;
10234 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10235 0 : return fastEmitInst_rr(X86::VPMULHRSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10236 : }
10237 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10238 0 : return fastEmitInst_rr(X86::VPMULHRSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10239 : }
10240 : return 0;
10241 : }
10242 :
10243 : unsigned fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10244 0 : if (RetVT.SimpleTy != MVT::v32i16)
10245 : return 0;
10246 0 : if ((Subtarget->hasBWI())) {
10247 0 : return fastEmitInst_rr(X86::VPMULHRSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10248 : }
10249 : return 0;
10250 : }
10251 :
10252 0 : unsigned fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10253 0 : switch (VT.SimpleTy) {
10254 0 : case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10255 0 : case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10256 0 : case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10257 : default: return 0;
10258 : }
10259 : }
10260 :
10261 : // FastEmit functions for X86ISD::MULTISHIFT.
10262 :
10263 0 : unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10264 0 : if (RetVT.SimpleTy != MVT::v16i8)
10265 : return 0;
10266 0 : if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
10267 0 : return fastEmitInst_rr(X86::VPMULTISHIFTQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10268 : }
10269 : return 0;
10270 : }
10271 :
10272 0 : unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10273 0 : if (RetVT.SimpleTy != MVT::v32i8)
10274 : return 0;
10275 0 : if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
10276 0 : return fastEmitInst_rr(X86::VPMULTISHIFTQBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10277 : }
10278 : return 0;
10279 : }
10280 :
10281 : unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10282 0 : if (RetVT.SimpleTy != MVT::v64i8)
10283 : return 0;
10284 0 : if ((Subtarget->hasVBMI())) {
10285 0 : return fastEmitInst_rr(X86::VPMULTISHIFTQBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10286 : }
10287 : return 0;
10288 : }
10289 :
10290 0 : unsigned fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10291 0 : switch (VT.SimpleTy) {
10292 0 : case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10293 0 : case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10294 0 : case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10295 : default: return 0;
10296 : }
10297 : }
10298 :
10299 : // FastEmit functions for X86ISD::PACKSS.
10300 :
10301 0 : unsigned fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10302 0 : if (RetVT.SimpleTy != MVT::v16i8)
10303 : return 0;
10304 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10305 0 : return fastEmitInst_rr(X86::VPACKSSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10306 : }
10307 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10308 0 : return fastEmitInst_rr(X86::PACKSSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10309 : }
10310 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10311 0 : return fastEmitInst_rr(X86::VPACKSSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10312 : }
10313 : return 0;
10314 : }
10315 :
10316 0 : unsigned fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10317 0 : if (RetVT.SimpleTy != MVT::v32i8)
10318 : return 0;
10319 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10320 0 : return fastEmitInst_rr(X86::VPACKSSWBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10321 : }
10322 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10323 0 : return fastEmitInst_rr(X86::VPACKSSWBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10324 : }
10325 : return 0;
10326 : }
10327 :
10328 : unsigned fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10329 0 : if (RetVT.SimpleTy != MVT::v64i8)
10330 : return 0;
10331 0 : if ((Subtarget->hasBWI())) {
10332 0 : return fastEmitInst_rr(X86::VPACKSSWBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10333 : }
10334 : return 0;
10335 : }
10336 :
10337 0 : unsigned fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10338 0 : if (RetVT.SimpleTy != MVT::v8i16)
10339 : return 0;
10340 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10341 0 : return fastEmitInst_rr(X86::VPACKSSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10342 : }
10343 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10344 0 : return fastEmitInst_rr(X86::PACKSSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10345 : }
10346 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10347 0 : return fastEmitInst_rr(X86::VPACKSSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10348 : }
10349 : return 0;
10350 : }
10351 :
10352 0 : unsigned fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10353 0 : if (RetVT.SimpleTy != MVT::v16i16)
10354 : return 0;
10355 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10356 0 : return fastEmitInst_rr(X86::VPACKSSDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10357 : }
10358 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10359 0 : return fastEmitInst_rr(X86::VPACKSSDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10360 : }
10361 : return 0;
10362 : }
10363 :
10364 : unsigned fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10365 0 : if (RetVT.SimpleTy != MVT::v32i16)
10366 : return 0;
10367 0 : if ((Subtarget->hasBWI())) {
10368 0 : return fastEmitInst_rr(X86::VPACKSSDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10369 : }
10370 : return 0;
10371 : }
10372 :
10373 0 : unsigned fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10374 0 : switch (VT.SimpleTy) {
10375 0 : case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10376 0 : case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10377 0 : case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10378 0 : case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10379 0 : case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10380 0 : case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10381 : default: return 0;
10382 : }
10383 : }
10384 :
10385 : // FastEmit functions for X86ISD::PACKUS.
10386 :
10387 0 : unsigned fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10388 0 : if (RetVT.SimpleTy != MVT::v16i8)
10389 : return 0;
10390 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10391 0 : return fastEmitInst_rr(X86::VPACKUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10392 : }
10393 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10394 0 : return fastEmitInst_rr(X86::PACKUSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10395 : }
10396 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10397 0 : return fastEmitInst_rr(X86::VPACKUSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10398 : }
10399 : return 0;
10400 : }
10401 :
10402 0 : unsigned fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10403 0 : if (RetVT.SimpleTy != MVT::v32i8)
10404 : return 0;
10405 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10406 0 : return fastEmitInst_rr(X86::VPACKUSWBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10407 : }
10408 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10409 0 : return fastEmitInst_rr(X86::VPACKUSWBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10410 : }
10411 : return 0;
10412 : }
10413 :
10414 : unsigned fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10415 0 : if (RetVT.SimpleTy != MVT::v64i8)
10416 : return 0;
10417 0 : if ((Subtarget->hasBWI())) {
10418 0 : return fastEmitInst_rr(X86::VPACKUSWBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10419 : }
10420 : return 0;
10421 : }
10422 :
10423 0 : unsigned fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10424 0 : if (RetVT.SimpleTy != MVT::v8i16)
10425 : return 0;
10426 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10427 0 : return fastEmitInst_rr(X86::VPACKUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10428 : }
10429 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10430 0 : return fastEmitInst_rr(X86::PACKUSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10431 : }
10432 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10433 0 : return fastEmitInst_rr(X86::VPACKUSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10434 : }
10435 : return 0;
10436 : }
10437 :
10438 0 : unsigned fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10439 0 : if (RetVT.SimpleTy != MVT::v16i16)
10440 : return 0;
10441 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10442 0 : return fastEmitInst_rr(X86::VPACKUSDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10443 : }
10444 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10445 0 : return fastEmitInst_rr(X86::VPACKUSDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10446 : }
10447 : return 0;
10448 : }
10449 :
10450 : unsigned fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10451 0 : if (RetVT.SimpleTy != MVT::v32i16)
10452 : return 0;
10453 0 : if ((Subtarget->hasBWI())) {
10454 0 : return fastEmitInst_rr(X86::VPACKUSDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10455 : }
10456 : return 0;
10457 : }
10458 :
10459 0 : unsigned fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10460 0 : switch (VT.SimpleTy) {
10461 0 : case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10462 0 : case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10463 0 : case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10464 0 : case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10465 0 : case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10466 0 : case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10467 : default: return 0;
10468 : }
10469 : }
10470 :
10471 : // FastEmit functions for X86ISD::PCMPEQ.
10472 :
10473 0 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10474 0 : if (RetVT.SimpleTy != MVT::v16i8)
10475 : return 0;
10476 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10477 0 : return fastEmitInst_rr(X86::PCMPEQBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10478 : }
10479 0 : if ((Subtarget->hasAVX()) && (true)) {
10480 0 : return fastEmitInst_rr(X86::VPCMPEQBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10481 : }
10482 : return 0;
10483 : }
10484 :
10485 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10486 0 : if (RetVT.SimpleTy != MVT::v32i8)
10487 : return 0;
10488 0 : if ((Subtarget->hasAVX2()) && (true)) {
10489 0 : return fastEmitInst_rr(X86::VPCMPEQBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10490 : }
10491 : return 0;
10492 : }
10493 :
10494 0 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10495 0 : if (RetVT.SimpleTy != MVT::v8i16)
10496 : return 0;
10497 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10498 0 : return fastEmitInst_rr(X86::PCMPEQWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10499 : }
10500 0 : if ((Subtarget->hasAVX()) && (true)) {
10501 0 : return fastEmitInst_rr(X86::VPCMPEQWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10502 : }
10503 : return 0;
10504 : }
10505 :
10506 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10507 0 : if (RetVT.SimpleTy != MVT::v16i16)
10508 : return 0;
10509 0 : if ((Subtarget->hasAVX2()) && (true)) {
10510 0 : return fastEmitInst_rr(X86::VPCMPEQWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10511 : }
10512 : return 0;
10513 : }
10514 :
10515 0 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10516 0 : if (RetVT.SimpleTy != MVT::v4i32)
10517 : return 0;
10518 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10519 0 : return fastEmitInst_rr(X86::PCMPEQDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10520 : }
10521 0 : if ((Subtarget->hasAVX()) && (true)) {
10522 0 : return fastEmitInst_rr(X86::VPCMPEQDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10523 : }
10524 : return 0;
10525 : }
10526 :
10527 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10528 0 : if (RetVT.SimpleTy != MVT::v8i32)
10529 : return 0;
10530 0 : if ((Subtarget->hasAVX2()) && (true)) {
10531 0 : return fastEmitInst_rr(X86::VPCMPEQDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10532 : }
10533 : return 0;
10534 : }
10535 :
10536 0 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10537 0 : if (RetVT.SimpleTy != MVT::v2i64)
10538 : return 0;
10539 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10540 0 : return fastEmitInst_rr(X86::PCMPEQQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10541 : }
10542 0 : if ((Subtarget->hasAVX())) {
10543 0 : return fastEmitInst_rr(X86::VPCMPEQQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10544 : }
10545 : return 0;
10546 : }
10547 :
10548 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10549 0 : if (RetVT.SimpleTy != MVT::v4i64)
10550 : return 0;
10551 0 : if ((Subtarget->hasAVX2())) {
10552 0 : return fastEmitInst_rr(X86::VPCMPEQQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10553 : }
10554 : return 0;
10555 : }
10556 :
10557 0 : unsigned fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10558 0 : switch (VT.SimpleTy) {
10559 0 : case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10560 0 : case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10561 0 : case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10562 0 : case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10563 0 : case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10564 0 : case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10565 0 : case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10566 0 : case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10567 : default: return 0;
10568 : }
10569 : }
10570 :
10571 : // FastEmit functions for X86ISD::PCMPGT.
10572 :
10573 0 : unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10574 0 : if (RetVT.SimpleTy != MVT::v16i8)
10575 : return 0;
10576 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10577 0 : return fastEmitInst_rr(X86::PCMPGTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10578 : }
10579 0 : if ((Subtarget->hasAVX()) && (true)) {
10580 0 : return fastEmitInst_rr(X86::VPCMPGTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10581 : }
10582 : return 0;
10583 : }
10584 :
10585 : unsigned fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10586 0 : if (RetVT.SimpleTy != MVT::v32i8)
10587 : return 0;
10588 0 : if ((Subtarget->hasAVX2()) && (true)) {
10589 0 : return fastEmitInst_rr(X86::VPCMPGTBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10590 : }
10591 : return 0;
10592 : }
10593 :
10594 0 : unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10595 0 : if (RetVT.SimpleTy != MVT::v8i16)
10596 : return 0;
10597 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10598 0 : return fastEmitInst_rr(X86::PCMPGTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10599 : }
10600 0 : if ((Subtarget->hasAVX()) && (true)) {
10601 0 : return fastEmitInst_rr(X86::VPCMPGTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10602 : }
10603 : return 0;
10604 : }
10605 :
10606 : unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10607 0 : if (RetVT.SimpleTy != MVT::v16i16)
10608 : return 0;
10609 0 : if ((Subtarget->hasAVX2()) && (true)) {
10610 0 : return fastEmitInst_rr(X86::VPCMPGTWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10611 : }
10612 : return 0;
10613 : }
10614 :
10615 0 : unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10616 0 : if (RetVT.SimpleTy != MVT::v4i32)
10617 : return 0;
10618 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10619 0 : return fastEmitInst_rr(X86::PCMPGTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10620 : }
10621 0 : if ((Subtarget->hasAVX()) && (true)) {
10622 0 : return fastEmitInst_rr(X86::VPCMPGTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10623 : }
10624 : return 0;
10625 : }
10626 :
10627 : unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10628 0 : if (RetVT.SimpleTy != MVT::v8i32)
10629 : return 0;
10630 0 : if ((Subtarget->hasAVX2()) && (true)) {
10631 0 : return fastEmitInst_rr(X86::VPCMPGTDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10632 : }
10633 : return 0;
10634 : }
10635 :
10636 0 : unsigned fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10637 0 : if (RetVT.SimpleTy != MVT::v2i64)
10638 : return 0;
10639 0 : if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
10640 0 : return fastEmitInst_rr(X86::PCMPGTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10641 : }
10642 0 : if ((Subtarget->hasAVX())) {
10643 0 : return fastEmitInst_rr(X86::VPCMPGTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10644 : }
10645 : return 0;
10646 : }
10647 :
10648 : unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10649 0 : if (RetVT.SimpleTy != MVT::v4i64)
10650 : return 0;
10651 0 : if ((Subtarget->hasAVX2())) {
10652 0 : return fastEmitInst_rr(X86::VPCMPGTQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10653 : }
10654 : return 0;
10655 : }
10656 :
10657 0 : unsigned fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10658 0 : switch (VT.SimpleTy) {
10659 0 : case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10660 0 : case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10661 0 : case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10662 0 : case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10663 0 : case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10664 0 : case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10665 0 : case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10666 0 : case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10667 : default: return 0;
10668 : }
10669 : }
10670 :
10671 : // FastEmit functions for X86ISD::PMULDQ.
10672 :
10673 0 : unsigned fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10674 0 : if (RetVT.SimpleTy != MVT::v2i64)
10675 : return 0;
10676 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10677 0 : return fastEmitInst_rr(X86::VPMULDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10678 : }
10679 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10680 0 : return fastEmitInst_rr(X86::PMULDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10681 : }
10682 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10683 0 : return fastEmitInst_rr(X86::VPMULDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10684 : }
10685 : return 0;
10686 : }
10687 :
10688 0 : unsigned fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10689 0 : if (RetVT.SimpleTy != MVT::v4i64)
10690 : return 0;
10691 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10692 0 : return fastEmitInst_rr(X86::VPMULDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10693 : }
10694 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10695 0 : return fastEmitInst_rr(X86::VPMULDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10696 : }
10697 : return 0;
10698 : }
10699 :
10700 : unsigned fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10701 0 : if (RetVT.SimpleTy != MVT::v8i64)
10702 : return 0;
10703 0 : if ((Subtarget->hasAVX512())) {
10704 0 : return fastEmitInst_rr(X86::VPMULDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10705 : }
10706 : return 0;
10707 : }
10708 :
10709 0 : unsigned fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10710 0 : switch (VT.SimpleTy) {
10711 0 : case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10712 0 : case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10713 0 : case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10714 : default: return 0;
10715 : }
10716 : }
10717 :
10718 : // FastEmit functions for X86ISD::PMULUDQ.
10719 :
10720 0 : unsigned fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10721 0 : if (RetVT.SimpleTy != MVT::v2i64)
10722 : return 0;
10723 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10724 0 : return fastEmitInst_rr(X86::VPMULUDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10725 : }
10726 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10727 0 : return fastEmitInst_rr(X86::PMULUDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10728 : }
10729 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10730 0 : return fastEmitInst_rr(X86::VPMULUDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10731 : }
10732 : return 0;
10733 : }
10734 :
10735 0 : unsigned fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10736 0 : if (RetVT.SimpleTy != MVT::v4i64)
10737 : return 0;
10738 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10739 0 : return fastEmitInst_rr(X86::VPMULUDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10740 : }
10741 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10742 0 : return fastEmitInst_rr(X86::VPMULUDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10743 : }
10744 : return 0;
10745 : }
10746 :
10747 : unsigned fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10748 0 : if (RetVT.SimpleTy != MVT::v8i64)
10749 : return 0;
10750 0 : if ((Subtarget->hasAVX512())) {
10751 0 : return fastEmitInst_rr(X86::VPMULUDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10752 : }
10753 : return 0;
10754 : }
10755 :
10756 0 : unsigned fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10757 0 : switch (VT.SimpleTy) {
10758 0 : case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10759 0 : case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10760 0 : case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10761 : default: return 0;
10762 : }
10763 : }
10764 :
10765 : // FastEmit functions for X86ISD::PSADBW.
10766 :
10767 0 : unsigned fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10768 0 : if (RetVT.SimpleTy != MVT::v2i64)
10769 : return 0;
10770 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10771 0 : return fastEmitInst_rr(X86::VPSADBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10772 : }
10773 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10774 0 : return fastEmitInst_rr(X86::PSADBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10775 : }
10776 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10777 0 : return fastEmitInst_rr(X86::VPSADBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10778 : }
10779 : return 0;
10780 : }
10781 :
10782 0 : unsigned fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10783 0 : if (RetVT.SimpleTy != MVT::v4i64)
10784 : return 0;
10785 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10786 0 : return fastEmitInst_rr(X86::VPSADBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10787 : }
10788 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10789 0 : return fastEmitInst_rr(X86::VPSADBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10790 : }
10791 : return 0;
10792 : }
10793 :
10794 : unsigned fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10795 0 : if (RetVT.SimpleTy != MVT::v8i64)
10796 : return 0;
10797 0 : if ((Subtarget->hasBWI())) {
10798 0 : return fastEmitInst_rr(X86::VPSADBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10799 : }
10800 : return 0;
10801 : }
10802 :
10803 0 : unsigned fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10804 0 : switch (VT.SimpleTy) {
10805 0 : case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10806 0 : case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10807 0 : case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10808 : default: return 0;
10809 : }
10810 : }
10811 :
10812 : // FastEmit functions for X86ISD::PSHUFB.
10813 :
10814 0 : unsigned fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10815 0 : if (RetVT.SimpleTy != MVT::v16i8)
10816 : return 0;
10817 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10818 0 : return fastEmitInst_rr(X86::VPSHUFBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10819 : }
10820 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
10821 0 : return fastEmitInst_rr(X86::PSHUFBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10822 : }
10823 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10824 0 : return fastEmitInst_rr(X86::VPSHUFBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10825 : }
10826 : return 0;
10827 : }
10828 :
10829 0 : unsigned fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10830 0 : if (RetVT.SimpleTy != MVT::v32i8)
10831 : return 0;
10832 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10833 0 : return fastEmitInst_rr(X86::VPSHUFBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10834 : }
10835 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10836 0 : return fastEmitInst_rr(X86::VPSHUFBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10837 : }
10838 : return 0;
10839 : }
10840 :
10841 : unsigned fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10842 0 : if (RetVT.SimpleTy != MVT::v64i8)
10843 : return 0;
10844 0 : if ((Subtarget->hasBWI())) {
10845 0 : return fastEmitInst_rr(X86::VPSHUFBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10846 : }
10847 : return 0;
10848 : }
10849 :
10850 0 : unsigned fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10851 0 : switch (VT.SimpleTy) {
10852 0 : case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10853 0 : case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10854 0 : case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10855 : default: return 0;
10856 : }
10857 : }
10858 :
10859 : // FastEmit functions for X86ISD::PTEST.
10860 :
10861 0 : unsigned fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10862 0 : if (RetVT.SimpleTy != MVT::i32)
10863 : return 0;
10864 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10865 0 : return fastEmitInst_rr(X86::PTESTrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10866 : }
10867 0 : if ((Subtarget->hasAVX())) {
10868 0 : return fastEmitInst_rr(X86::VPTESTrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10869 : }
10870 : return 0;
10871 : }
10872 :
10873 : unsigned fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10874 0 : if (RetVT.SimpleTy != MVT::i32)
10875 : return 0;
10876 0 : if ((Subtarget->hasAVX())) {
10877 0 : return fastEmitInst_rr(X86::VPTESTYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10878 : }
10879 : return 0;
10880 : }
10881 :
10882 0 : unsigned fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10883 0 : switch (VT.SimpleTy) {
10884 0 : case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10885 0 : case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10886 : default: return 0;
10887 : }
10888 : }
10889 :
10890 : // FastEmit functions for X86ISD::RCP14S.
10891 :
10892 : unsigned fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10893 0 : if (RetVT.SimpleTy != MVT::v4f32)
10894 : return 0;
10895 0 : if ((Subtarget->hasAVX512())) {
10896 0 : return fastEmitInst_rr(X86::VRCP14SSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10897 : }
10898 : return 0;
10899 : }
10900 :
10901 : unsigned fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10902 0 : if (RetVT.SimpleTy != MVT::v2f64)
10903 : return 0;
10904 0 : if ((Subtarget->hasAVX512())) {
10905 0 : return fastEmitInst_rr(X86::VRCP14SDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10906 : }
10907 : return 0;
10908 : }
10909 :
10910 0 : unsigned fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10911 0 : switch (VT.SimpleTy) {
10912 0 : case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10913 0 : case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10914 : default: return 0;
10915 : }
10916 : }
10917 :
10918 : // FastEmit functions for X86ISD::RSQRT14S.
10919 :
10920 : unsigned fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10921 0 : if (RetVT.SimpleTy != MVT::v4f32)
10922 : return 0;
10923 0 : if ((Subtarget->hasAVX512())) {
10924 0 : return fastEmitInst_rr(X86::VRSQRT14SSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10925 : }
10926 : return 0;
10927 : }
10928 :
10929 : unsigned fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10930 0 : if (RetVT.SimpleTy != MVT::v2f64)
10931 : return 0;
10932 0 : if ((Subtarget->hasAVX512())) {
10933 0 : return fastEmitInst_rr(X86::VRSQRT14SDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10934 : }
10935 : return 0;
10936 : }
10937 :
10938 0 : unsigned fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10939 0 : switch (VT.SimpleTy) {
10940 0 : case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10941 0 : case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10942 : default: return 0;
10943 : }
10944 : }
10945 :
10946 : // FastEmit functions for X86ISD::SUBS.
10947 :
10948 0 : unsigned fastEmit_X86ISD_SUBS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10949 0 : if (RetVT.SimpleTy != MVT::v16i8)
10950 : return 0;
10951 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10952 0 : return fastEmitInst_rr(X86::VPSUBSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10953 : }
10954 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10955 0 : return fastEmitInst_rr(X86::PSUBSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10956 : }
10957 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10958 0 : return fastEmitInst_rr(X86::VPSUBSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10959 : }
10960 : return 0;
10961 : }
10962 :
10963 0 : unsigned fastEmit_X86ISD_SUBS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10964 0 : if (RetVT.SimpleTy != MVT::v32i8)
10965 : return 0;
10966 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10967 0 : return fastEmitInst_rr(X86::VPSUBSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10968 : }
10969 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10970 0 : return fastEmitInst_rr(X86::VPSUBSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10971 : }
10972 : return 0;
10973 : }
10974 :
10975 : unsigned fastEmit_X86ISD_SUBS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10976 0 : if (RetVT.SimpleTy != MVT::v64i8)
10977 : return 0;
10978 0 : if ((Subtarget->hasBWI())) {
10979 0 : return fastEmitInst_rr(X86::VPSUBSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10980 : }
10981 : return 0;
10982 : }
10983 :
10984 0 : unsigned fastEmit_X86ISD_SUBS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10985 0 : if (RetVT.SimpleTy != MVT::v8i16)
10986 : return 0;
10987 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10988 0 : return fastEmitInst_rr(X86::VPSUBSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10989 : }
10990 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10991 0 : return fastEmitInst_rr(X86::PSUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10992 : }
10993 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10994 0 : return fastEmitInst_rr(X86::VPSUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10995 : }
10996 : return 0;
10997 : }
10998 :
10999 0 : unsigned fastEmit_X86ISD_SUBS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11000 0 : if (RetVT.SimpleTy != MVT::v16i16)
11001 : return 0;
11002 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11003 0 : return fastEmitInst_rr(X86::VPSUBSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11004 : }
11005 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11006 0 : return fastEmitInst_rr(X86::VPSUBSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11007 : }
11008 : return 0;
11009 : }
11010 :
11011 : unsigned fastEmit_X86ISD_SUBS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11012 0 : if (RetVT.SimpleTy != MVT::v32i16)
11013 : return 0;
11014 0 : if ((Subtarget->hasBWI())) {
11015 0 : return fastEmitInst_rr(X86::VPSUBSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11016 : }
11017 : return 0;
11018 : }
11019 :
11020 0 : unsigned fastEmit_X86ISD_SUBS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11021 0 : switch (VT.SimpleTy) {
11022 0 : case MVT::v16i8: return fastEmit_X86ISD_SUBS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11023 0 : case MVT::v32i8: return fastEmit_X86ISD_SUBS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11024 0 : case MVT::v64i8: return fastEmit_X86ISD_SUBS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11025 0 : case MVT::v8i16: return fastEmit_X86ISD_SUBS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11026 0 : case MVT::v16i16: return fastEmit_X86ISD_SUBS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11027 0 : case MVT::v32i16: return fastEmit_X86ISD_SUBS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11028 : default: return 0;
11029 : }
11030 : }
11031 :
11032 : // FastEmit functions for X86ISD::SUBUS.
11033 :
11034 0 : unsigned fastEmit_X86ISD_SUBUS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11035 0 : if (RetVT.SimpleTy != MVT::v16i8)
11036 : return 0;
11037 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11038 0 : return fastEmitInst_rr(X86::VPSUBUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11039 : }
11040 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11041 0 : return fastEmitInst_rr(X86::PSUBUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11042 : }
11043 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11044 0 : return fastEmitInst_rr(X86::VPSUBUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11045 : }
11046 : return 0;
11047 : }
11048 :
11049 0 : unsigned fastEmit_X86ISD_SUBUS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11050 0 : if (RetVT.SimpleTy != MVT::v32i8)
11051 : return 0;
11052 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11053 0 : return fastEmitInst_rr(X86::VPSUBUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11054 : }
11055 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11056 0 : return fastEmitInst_rr(X86::VPSUBUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11057 : }
11058 : return 0;
11059 : }
11060 :
11061 : unsigned fastEmit_X86ISD_SUBUS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11062 0 : if (RetVT.SimpleTy != MVT::v64i8)
11063 : return 0;
11064 0 : if ((Subtarget->hasBWI())) {
11065 0 : return fastEmitInst_rr(X86::VPSUBUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11066 : }
11067 : return 0;
11068 : }
11069 :
11070 0 : unsigned fastEmit_X86ISD_SUBUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11071 0 : if (RetVT.SimpleTy != MVT::v8i16)
11072 : return 0;
11073 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11074 0 : return fastEmitInst_rr(X86::VPSUBUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11075 : }
11076 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11077 0 : return fastEmitInst_rr(X86::PSUBUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11078 : }
11079 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11080 0 : return fastEmitInst_rr(X86::VPSUBUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11081 : }
11082 : return 0;
11083 : }
11084 :
11085 0 : unsigned fastEmit_X86ISD_SUBUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11086 0 : if (RetVT.SimpleTy != MVT::v16i16)
11087 : return 0;
11088 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11089 0 : return fastEmitInst_rr(X86::VPSUBUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11090 : }
11091 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11092 0 : return fastEmitInst_rr(X86::VPSUBUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11093 : }
11094 : return 0;
11095 : }
11096 :
11097 : unsigned fastEmit_X86ISD_SUBUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11098 0 : if (RetVT.SimpleTy != MVT::v32i16)
11099 : return 0;
11100 0 : if ((Subtarget->hasBWI())) {
11101 0 : return fastEmitInst_rr(X86::VPSUBUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11102 : }
11103 : return 0;
11104 : }
11105 :
11106 0 : unsigned fastEmit_X86ISD_SUBUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11107 0 : switch (VT.SimpleTy) {
11108 0 : case MVT::v16i8: return fastEmit_X86ISD_SUBUS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11109 0 : case MVT::v32i8: return fastEmit_X86ISD_SUBUS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11110 0 : case MVT::v64i8: return fastEmit_X86ISD_SUBUS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11111 0 : case MVT::v8i16: return fastEmit_X86ISD_SUBUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11112 0 : case MVT::v16i16: return fastEmit_X86ISD_SUBUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11113 0 : case MVT::v32i16: return fastEmit_X86ISD_SUBUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11114 : default: return 0;
11115 : }
11116 : }
11117 :
11118 : // FastEmit functions for X86ISD::TESTP.
11119 :
11120 : unsigned fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11121 0 : if (RetVT.SimpleTy != MVT::i32)
11122 : return 0;
11123 0 : if ((Subtarget->hasAVX())) {
11124 0 : return fastEmitInst_rr(X86::VTESTPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11125 : }
11126 : return 0;
11127 : }
11128 :
11129 : unsigned fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11130 0 : if (RetVT.SimpleTy != MVT::i32)
11131 : return 0;
11132 0 : if ((Subtarget->hasAVX())) {
11133 0 : return fastEmitInst_rr(X86::VTESTPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11134 : }
11135 : return 0;
11136 : }
11137 :
11138 : unsigned fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11139 0 : if (RetVT.SimpleTy != MVT::i32)
11140 : return 0;
11141 0 : if ((Subtarget->hasAVX())) {
11142 0 : return fastEmitInst_rr(X86::VTESTPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11143 : }
11144 : return 0;
11145 : }
11146 :
11147 : unsigned fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11148 0 : if (RetVT.SimpleTy != MVT::i32)
11149 : return 0;
11150 0 : if ((Subtarget->hasAVX())) {
11151 0 : return fastEmitInst_rr(X86::VTESTPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11152 : }
11153 : return 0;
11154 : }
11155 :
11156 0 : unsigned fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11157 0 : switch (VT.SimpleTy) {
11158 0 : case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11159 0 : case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11160 0 : case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11161 0 : case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11162 : default: return 0;
11163 : }
11164 : }
11165 :
11166 : // FastEmit functions for X86ISD::UCOMI.
11167 :
11168 0 : unsigned fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11169 0 : if (RetVT.SimpleTy != MVT::i32)
11170 : return 0;
11171 0 : if ((Subtarget->hasAVX512())) {
11172 0 : return fastEmitInst_rr(X86::VUCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11173 : }
11174 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11175 0 : return fastEmitInst_rr(X86::UCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11176 : }
11177 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11178 0 : return fastEmitInst_rr(X86::VUCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11179 : }
11180 : return 0;
11181 : }
11182 :
11183 0 : unsigned fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11184 0 : if (RetVT.SimpleTy != MVT::i32)
11185 : return 0;
11186 0 : if ((Subtarget->hasAVX512())) {
11187 0 : return fastEmitInst_rr(X86::VUCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11188 : }
11189 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11190 0 : return fastEmitInst_rr(X86::UCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11191 : }
11192 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11193 0 : return fastEmitInst_rr(X86::VUCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11194 : }
11195 : return 0;
11196 : }
11197 :
11198 0 : unsigned fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11199 0 : switch (VT.SimpleTy) {
11200 0 : case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11201 0 : case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11202 : default: return 0;
11203 : }
11204 : }
11205 :
11206 : // FastEmit functions for X86ISD::UNPCKH.
11207 :
11208 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11209 0 : if (RetVT.SimpleTy != MVT::v16i8)
11210 : return 0;
11211 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11212 0 : return fastEmitInst_rr(X86::VPUNPCKHBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11213 : }
11214 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11215 0 : return fastEmitInst_rr(X86::PUNPCKHBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11216 : }
11217 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11218 0 : return fastEmitInst_rr(X86::VPUNPCKHBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11219 : }
11220 : return 0;
11221 : }
11222 :
11223 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11224 0 : if (RetVT.SimpleTy != MVT::v32i8)
11225 : return 0;
11226 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11227 0 : return fastEmitInst_rr(X86::VPUNPCKHBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11228 : }
11229 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11230 0 : return fastEmitInst_rr(X86::VPUNPCKHBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11231 : }
11232 : return 0;
11233 : }
11234 :
11235 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11236 0 : if (RetVT.SimpleTy != MVT::v64i8)
11237 : return 0;
11238 0 : if ((Subtarget->hasBWI())) {
11239 0 : return fastEmitInst_rr(X86::VPUNPCKHBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11240 : }
11241 : return 0;
11242 : }
11243 :
11244 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11245 0 : if (RetVT.SimpleTy != MVT::v8i16)
11246 : return 0;
11247 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11248 0 : return fastEmitInst_rr(X86::VPUNPCKHWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11249 : }
11250 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11251 0 : return fastEmitInst_rr(X86::PUNPCKHWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11252 : }
11253 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11254 0 : return fastEmitInst_rr(X86::VPUNPCKHWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11255 : }
11256 : return 0;
11257 : }
11258 :
11259 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11260 0 : if (RetVT.SimpleTy != MVT::v16i16)
11261 : return 0;
11262 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11263 0 : return fastEmitInst_rr(X86::VPUNPCKHWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11264 : }
11265 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11266 0 : return fastEmitInst_rr(X86::VPUNPCKHWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11267 : }
11268 : return 0;
11269 : }
11270 :
11271 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11272 0 : if (RetVT.SimpleTy != MVT::v32i16)
11273 : return 0;
11274 0 : if ((Subtarget->hasBWI())) {
11275 0 : return fastEmitInst_rr(X86::VPUNPCKHWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11276 : }
11277 : return 0;
11278 : }
11279 :
11280 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11281 0 : if (RetVT.SimpleTy != MVT::v4i32)
11282 : return 0;
11283 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11284 0 : return fastEmitInst_rr(X86::VPUNPCKHDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11285 : }
11286 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11287 0 : return fastEmitInst_rr(X86::PUNPCKHDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11288 : }
11289 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11290 0 : return fastEmitInst_rr(X86::VPUNPCKHDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11291 : }
11292 : return 0;
11293 : }
11294 :
11295 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11296 0 : if (RetVT.SimpleTy != MVT::v8i32)
11297 : return 0;
11298 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
11299 0 : return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11300 : }
11301 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11302 0 : return fastEmitInst_rr(X86::VPUNPCKHDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11303 : }
11304 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11305 0 : return fastEmitInst_rr(X86::VPUNPCKHDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11306 : }
11307 : return 0;
11308 : }
11309 :
11310 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11311 0 : if (RetVT.SimpleTy != MVT::v16i32)
11312 : return 0;
11313 0 : if ((Subtarget->hasAVX512())) {
11314 0 : return fastEmitInst_rr(X86::VPUNPCKHDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11315 : }
11316 : return 0;
11317 : }
11318 :
11319 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11320 0 : if (RetVT.SimpleTy != MVT::v2i64)
11321 : return 0;
11322 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11323 0 : return fastEmitInst_rr(X86::VPUNPCKHQDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11324 : }
11325 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11326 0 : return fastEmitInst_rr(X86::PUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11327 : }
11328 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11329 0 : return fastEmitInst_rr(X86::VPUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11330 : }
11331 : return 0;
11332 : }
11333 :
11334 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11335 0 : if (RetVT.SimpleTy != MVT::v4i64)
11336 : return 0;
11337 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
11338 0 : return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11339 : }
11340 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11341 0 : return fastEmitInst_rr(X86::VPUNPCKHQDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11342 : }
11343 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11344 0 : return fastEmitInst_rr(X86::VPUNPCKHQDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11345 : }
11346 : return 0;
11347 : }
11348 :
11349 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11350 0 : if (RetVT.SimpleTy != MVT::v8i64)
11351 : return 0;
11352 0 : if ((Subtarget->hasAVX512())) {
11353 0 : return fastEmitInst_rr(X86::VPUNPCKHQDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11354 : }
11355 : return 0;
11356 : }
11357 :
11358 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11359 0 : if (RetVT.SimpleTy != MVT::v4f32)
11360 : return 0;
11361 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11362 0 : return fastEmitInst_rr(X86::VUNPCKHPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11363 : }
11364 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11365 0 : return fastEmitInst_rr(X86::UNPCKHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11366 : }
11367 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11368 0 : return fastEmitInst_rr(X86::VUNPCKHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11369 : }
11370 : return 0;
11371 : }
11372 :
11373 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11374 0 : if (RetVT.SimpleTy != MVT::v8f32)
11375 : return 0;
11376 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11377 0 : return fastEmitInst_rr(X86::VUNPCKHPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11378 : }
11379 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11380 0 : return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11381 : }
11382 : return 0;
11383 : }
11384 :
11385 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11386 0 : if (RetVT.SimpleTy != MVT::v16f32)
11387 : return 0;
11388 0 : if ((Subtarget->hasAVX512())) {
11389 0 : return fastEmitInst_rr(X86::VUNPCKHPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11390 : }
11391 : return 0;
11392 : }
11393 :
11394 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11395 0 : if (RetVT.SimpleTy != MVT::v2f64)
11396 : return 0;
11397 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11398 0 : return fastEmitInst_rr(X86::VUNPCKHPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11399 : }
11400 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11401 0 : return fastEmitInst_rr(X86::UNPCKHPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11402 : }
11403 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11404 0 : return fastEmitInst_rr(X86::VUNPCKHPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11405 : }
11406 : return 0;
11407 : }
11408 :
11409 0 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11410 0 : if (RetVT.SimpleTy != MVT::v4f64)
11411 : return 0;
11412 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11413 0 : return fastEmitInst_rr(X86::VUNPCKHPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11414 : }
11415 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11416 0 : return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11417 : }
11418 : return 0;
11419 : }
11420 :
11421 : unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11422 0 : if (RetVT.SimpleTy != MVT::v8f64)
11423 : return 0;
11424 0 : if ((Subtarget->hasAVX512())) {
11425 0 : return fastEmitInst_rr(X86::VUNPCKHPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11426 : }
11427 : return 0;
11428 : }
11429 :
11430 0 : unsigned fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11431 0 : switch (VT.SimpleTy) {
11432 0 : case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11433 0 : case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11434 0 : case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11435 0 : case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11436 0 : case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11437 0 : case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11438 0 : case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11439 0 : case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11440 0 : case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11441 0 : case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11442 0 : case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11443 0 : case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11444 0 : case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11445 0 : case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11446 0 : case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11447 0 : case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11448 0 : case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11449 0 : case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11450 : default: return 0;
11451 : }
11452 : }
11453 :
11454 : // FastEmit functions for X86ISD::UNPCKL.
11455 :
11456 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11457 0 : if (RetVT.SimpleTy != MVT::v16i8)
11458 : return 0;
11459 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11460 0 : return fastEmitInst_rr(X86::VPUNPCKLBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11461 : }
11462 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11463 0 : return fastEmitInst_rr(X86::PUNPCKLBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11464 : }
11465 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11466 0 : return fastEmitInst_rr(X86::VPUNPCKLBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11467 : }
11468 : return 0;
11469 : }
11470 :
11471 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11472 0 : if (RetVT.SimpleTy != MVT::v32i8)
11473 : return 0;
11474 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11475 0 : return fastEmitInst_rr(X86::VPUNPCKLBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11476 : }
11477 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11478 0 : return fastEmitInst_rr(X86::VPUNPCKLBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11479 : }
11480 : return 0;
11481 : }
11482 :
11483 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11484 0 : if (RetVT.SimpleTy != MVT::v64i8)
11485 : return 0;
11486 0 : if ((Subtarget->hasBWI())) {
11487 0 : return fastEmitInst_rr(X86::VPUNPCKLBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11488 : }
11489 : return 0;
11490 : }
11491 :
11492 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11493 0 : if (RetVT.SimpleTy != MVT::v8i16)
11494 : return 0;
11495 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11496 0 : return fastEmitInst_rr(X86::VPUNPCKLWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11497 : }
11498 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11499 0 : return fastEmitInst_rr(X86::PUNPCKLWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11500 : }
11501 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11502 0 : return fastEmitInst_rr(X86::VPUNPCKLWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11503 : }
11504 : return 0;
11505 : }
11506 :
11507 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11508 0 : if (RetVT.SimpleTy != MVT::v16i16)
11509 : return 0;
11510 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11511 0 : return fastEmitInst_rr(X86::VPUNPCKLWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11512 : }
11513 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11514 0 : return fastEmitInst_rr(X86::VPUNPCKLWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11515 : }
11516 : return 0;
11517 : }
11518 :
11519 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11520 0 : if (RetVT.SimpleTy != MVT::v32i16)
11521 : return 0;
11522 0 : if ((Subtarget->hasBWI())) {
11523 0 : return fastEmitInst_rr(X86::VPUNPCKLWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11524 : }
11525 : return 0;
11526 : }
11527 :
11528 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11529 0 : if (RetVT.SimpleTy != MVT::v4i32)
11530 : return 0;
11531 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11532 0 : return fastEmitInst_rr(X86::VPUNPCKLDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11533 : }
11534 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11535 0 : return fastEmitInst_rr(X86::PUNPCKLDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11536 : }
11537 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11538 0 : return fastEmitInst_rr(X86::VPUNPCKLDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11539 : }
11540 : return 0;
11541 : }
11542 :
11543 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11544 0 : if (RetVT.SimpleTy != MVT::v8i32)
11545 : return 0;
11546 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
11547 0 : return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11548 : }
11549 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11550 0 : return fastEmitInst_rr(X86::VPUNPCKLDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11551 : }
11552 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11553 0 : return fastEmitInst_rr(X86::VPUNPCKLDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11554 : }
11555 : return 0;
11556 : }
11557 :
11558 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11559 0 : if (RetVT.SimpleTy != MVT::v16i32)
11560 : return 0;
11561 0 : if ((Subtarget->hasAVX512())) {
11562 0 : return fastEmitInst_rr(X86::VPUNPCKLDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11563 : }
11564 : return 0;
11565 : }
11566 :
11567 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11568 0 : if (RetVT.SimpleTy != MVT::v2i64)
11569 : return 0;
11570 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11571 0 : return fastEmitInst_rr(X86::VPUNPCKLQDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11572 : }
11573 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11574 0 : return fastEmitInst_rr(X86::PUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11575 : }
11576 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11577 0 : return fastEmitInst_rr(X86::VPUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11578 : }
11579 : return 0;
11580 : }
11581 :
11582 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11583 0 : if (RetVT.SimpleTy != MVT::v4i64)
11584 : return 0;
11585 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
11586 0 : return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11587 : }
11588 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11589 0 : return fastEmitInst_rr(X86::VPUNPCKLQDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11590 : }
11591 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11592 0 : return fastEmitInst_rr(X86::VPUNPCKLQDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11593 : }
11594 : return 0;
11595 : }
11596 :
11597 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11598 0 : if (RetVT.SimpleTy != MVT::v8i64)
11599 : return 0;
11600 0 : if ((Subtarget->hasAVX512())) {
11601 0 : return fastEmitInst_rr(X86::VPUNPCKLQDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11602 : }
11603 : return 0;
11604 : }
11605 :
11606 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11607 0 : if (RetVT.SimpleTy != MVT::v4f32)
11608 : return 0;
11609 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11610 0 : return fastEmitInst_rr(X86::VUNPCKLPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11611 : }
11612 0 : if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11613 0 : return fastEmitInst_rr(X86::UNPCKLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11614 : }
11615 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11616 0 : return fastEmitInst_rr(X86::VUNPCKLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11617 : }
11618 : return 0;
11619 : }
11620 :
11621 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11622 0 : if (RetVT.SimpleTy != MVT::v8f32)
11623 : return 0;
11624 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11625 0 : return fastEmitInst_rr(X86::VUNPCKLPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11626 : }
11627 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11628 0 : return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11629 : }
11630 : return 0;
11631 : }
11632 :
11633 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11634 0 : if (RetVT.SimpleTy != MVT::v16f32)
11635 : return 0;
11636 0 : if ((Subtarget->hasAVX512())) {
11637 0 : return fastEmitInst_rr(X86::VUNPCKLPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11638 : }
11639 : return 0;
11640 : }
11641 :
11642 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11643 0 : if (RetVT.SimpleTy != MVT::v2f64)
11644 : return 0;
11645 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11646 0 : return fastEmitInst_rr(X86::VUNPCKLPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11647 : }
11648 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11649 0 : return fastEmitInst_rr(X86::UNPCKLPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11650 : }
11651 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11652 0 : return fastEmitInst_rr(X86::VUNPCKLPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11653 : }
11654 : return 0;
11655 : }
11656 :
11657 0 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11658 0 : if (RetVT.SimpleTy != MVT::v4f64)
11659 : return 0;
11660 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11661 0 : return fastEmitInst_rr(X86::VUNPCKLPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11662 : }
11663 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11664 0 : return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11665 : }
11666 : return 0;
11667 : }
11668 :
11669 : unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11670 0 : if (RetVT.SimpleTy != MVT::v8f64)
11671 : return 0;
11672 0 : if ((Subtarget->hasAVX512())) {
11673 0 : return fastEmitInst_rr(X86::VUNPCKLPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11674 : }
11675 : return 0;
11676 : }
11677 :
11678 0 : unsigned fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11679 0 : switch (VT.SimpleTy) {
11680 0 : case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11681 0 : case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11682 0 : case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11683 0 : case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11684 0 : case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11685 0 : case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11686 0 : case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11687 0 : case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11688 0 : case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11689 0 : case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11690 0 : case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11691 0 : case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11692 0 : case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11693 0 : case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11694 0 : case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11695 0 : case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11696 0 : case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11697 0 : case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11698 : default: return 0;
11699 : }
11700 : }
11701 :
11702 : // FastEmit functions for X86ISD::VPERMV.
11703 :
11704 0 : unsigned fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11705 0 : if (RetVT.SimpleTy != MVT::v16i8)
11706 : return 0;
11707 0 : if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
11708 0 : return fastEmitInst_rr(X86::VPERMBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11709 : }
11710 : return 0;
11711 : }
11712 :
11713 0 : unsigned fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11714 0 : if (RetVT.SimpleTy != MVT::v32i8)
11715 : return 0;
11716 0 : if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
11717 0 : return fastEmitInst_rr(X86::VPERMBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11718 : }
11719 : return 0;
11720 : }
11721 :
11722 : unsigned fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11723 0 : if (RetVT.SimpleTy != MVT::v64i8)
11724 : return 0;
11725 0 : if ((Subtarget->hasVBMI())) {
11726 0 : return fastEmitInst_rr(X86::VPERMBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11727 : }
11728 : return 0;
11729 : }
11730 :
11731 0 : unsigned fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11732 0 : if (RetVT.SimpleTy != MVT::v8i16)
11733 : return 0;
11734 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11735 0 : return fastEmitInst_rr(X86::VPERMWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11736 : }
11737 : return 0;
11738 : }
11739 :
11740 0 : unsigned fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11741 0 : if (RetVT.SimpleTy != MVT::v16i16)
11742 : return 0;
11743 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11744 0 : return fastEmitInst_rr(X86::VPERMWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11745 : }
11746 : return 0;
11747 : }
11748 :
11749 : unsigned fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11750 0 : if (RetVT.SimpleTy != MVT::v32i16)
11751 : return 0;
11752 0 : if ((Subtarget->hasBWI())) {
11753 0 : return fastEmitInst_rr(X86::VPERMWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11754 : }
11755 : return 0;
11756 : }
11757 :
11758 0 : unsigned fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11759 0 : if (RetVT.SimpleTy != MVT::v8i32)
11760 : return 0;
11761 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11762 0 : return fastEmitInst_rr(X86::VPERMDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11763 : }
11764 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
11765 0 : return fastEmitInst_rr(X86::VPERMDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11766 : }
11767 : return 0;
11768 : }
11769 :
11770 : unsigned fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11771 0 : if (RetVT.SimpleTy != MVT::v16i32)
11772 : return 0;
11773 0 : if ((Subtarget->hasAVX512())) {
11774 0 : return fastEmitInst_rr(X86::VPERMDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11775 : }
11776 : return 0;
11777 : }
11778 :
11779 0 : unsigned fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11780 0 : if (RetVT.SimpleTy != MVT::v4i64)
11781 : return 0;
11782 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11783 0 : return fastEmitInst_rr(X86::VPERMQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11784 : }
11785 : return 0;
11786 : }
11787 :
11788 : unsigned fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11789 0 : if (RetVT.SimpleTy != MVT::v8i64)
11790 : return 0;
11791 0 : if ((Subtarget->hasAVX512())) {
11792 0 : return fastEmitInst_rr(X86::VPERMQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11793 : }
11794 : return 0;
11795 : }
11796 :
11797 0 : unsigned fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11798 0 : switch (VT.SimpleTy) {
11799 0 : case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11800 0 : case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11801 0 : case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11802 0 : case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11803 0 : case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11804 0 : case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11805 0 : case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11806 0 : case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11807 0 : case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11808 0 : case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11809 : default: return 0;
11810 : }
11811 : }
11812 :
11813 : // FastEmit functions for X86ISD::VPMADDUBSW.
11814 :
11815 0 : unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11816 0 : if (RetVT.SimpleTy != MVT::v8i16)
11817 : return 0;
11818 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11819 0 : return fastEmitInst_rr(X86::VPMADDUBSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11820 : }
11821 0 : if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
11822 0 : return fastEmitInst_rr(X86::PMADDUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11823 : }
11824 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11825 0 : return fastEmitInst_rr(X86::VPMADDUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11826 : }
11827 : return 0;
11828 : }
11829 :
11830 0 : unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11831 0 : if (RetVT.SimpleTy != MVT::v16i16)
11832 : return 0;
11833 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11834 0 : return fastEmitInst_rr(X86::VPMADDUBSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11835 : }
11836 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11837 0 : return fastEmitInst_rr(X86::VPMADDUBSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11838 : }
11839 : return 0;
11840 : }
11841 :
11842 : unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11843 0 : if (RetVT.SimpleTy != MVT::v32i16)
11844 : return 0;
11845 0 : if ((Subtarget->hasBWI())) {
11846 0 : return fastEmitInst_rr(X86::VPMADDUBSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11847 : }
11848 : return 0;
11849 : }
11850 :
11851 0 : unsigned fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11852 0 : switch (VT.SimpleTy) {
11853 0 : case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11854 0 : case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11855 0 : case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11856 : default: return 0;
11857 : }
11858 : }
11859 :
11860 : // FastEmit functions for X86ISD::VPMADDWD.
11861 :
11862 0 : unsigned fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11863 0 : if (RetVT.SimpleTy != MVT::v4i32)
11864 : return 0;
11865 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11866 0 : return fastEmitInst_rr(X86::VPMADDWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11867 : }
11868 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11869 0 : return fastEmitInst_rr(X86::PMADDWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11870 : }
11871 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11872 0 : return fastEmitInst_rr(X86::VPMADDWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11873 : }
11874 : return 0;
11875 : }
11876 :
11877 0 : unsigned fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11878 0 : if (RetVT.SimpleTy != MVT::v8i32)
11879 : return 0;
11880 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
11881 0 : return fastEmitInst_rr(X86::VPMADDWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11882 : }
11883 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
11884 0 : return fastEmitInst_rr(X86::VPMADDWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11885 : }
11886 : return 0;
11887 : }
11888 :
11889 : unsigned fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11890 0 : if (RetVT.SimpleTy != MVT::v16i32)
11891 : return 0;
11892 0 : if ((Subtarget->hasBWI())) {
11893 0 : return fastEmitInst_rr(X86::VPMADDWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11894 : }
11895 : return 0;
11896 : }
11897 :
11898 0 : unsigned fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11899 0 : switch (VT.SimpleTy) {
11900 0 : case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11901 0 : case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11902 0 : case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11903 : default: return 0;
11904 : }
11905 : }
11906 :
11907 : // FastEmit functions for X86ISD::VPSHA.
11908 :
11909 : unsigned fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11910 0 : if (RetVT.SimpleTy != MVT::v16i8)
11911 : return 0;
11912 0 : if ((Subtarget->hasXOP())) {
11913 0 : return fastEmitInst_rr(X86::VPSHABrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11914 : }
11915 : return 0;
11916 : }
11917 :
11918 : unsigned fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11919 0 : if (RetVT.SimpleTy != MVT::v8i16)
11920 : return 0;
11921 0 : if ((Subtarget->hasXOP())) {
11922 0 : return fastEmitInst_rr(X86::VPSHAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11923 : }
11924 : return 0;
11925 : }
11926 :
11927 : unsigned fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11928 0 : if (RetVT.SimpleTy != MVT::v4i32)
11929 : return 0;
11930 0 : if ((Subtarget->hasXOP())) {
11931 0 : return fastEmitInst_rr(X86::VPSHADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11932 : }
11933 : return 0;
11934 : }
11935 :
11936 : unsigned fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11937 0 : if (RetVT.SimpleTy != MVT::v2i64)
11938 : return 0;
11939 0 : if ((Subtarget->hasXOP())) {
11940 0 : return fastEmitInst_rr(X86::VPSHAQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11941 : }
11942 : return 0;
11943 : }
11944 :
11945 0 : unsigned fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11946 0 : switch (VT.SimpleTy) {
11947 0 : case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11948 0 : case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11949 0 : case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11950 0 : case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11951 : default: return 0;
11952 : }
11953 : }
11954 :
11955 : // FastEmit functions for X86ISD::VPSHL.
11956 :
11957 : unsigned fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11958 0 : if (RetVT.SimpleTy != MVT::v16i8)
11959 : return 0;
11960 0 : if ((Subtarget->hasXOP())) {
11961 0 : return fastEmitInst_rr(X86::VPSHLBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11962 : }
11963 : return 0;
11964 : }
11965 :
11966 : unsigned fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11967 0 : if (RetVT.SimpleTy != MVT::v8i16)
11968 : return 0;
11969 0 : if ((Subtarget->hasXOP())) {
11970 0 : return fastEmitInst_rr(X86::VPSHLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11971 : }
11972 : return 0;
11973 : }
11974 :
11975 : unsigned fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11976 0 : if (RetVT.SimpleTy != MVT::v4i32)
11977 : return 0;
11978 0 : if ((Subtarget->hasXOP())) {
11979 0 : return fastEmitInst_rr(X86::VPSHLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11980 : }
11981 : return 0;
11982 : }
11983 :
11984 : unsigned fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11985 0 : if (RetVT.SimpleTy != MVT::v2i64)
11986 : return 0;
11987 0 : if ((Subtarget->hasXOP())) {
11988 0 : return fastEmitInst_rr(X86::VPSHLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11989 : }
11990 : return 0;
11991 : }
11992 :
11993 0 : unsigned fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11994 0 : switch (VT.SimpleTy) {
11995 0 : case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11996 0 : case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11997 0 : case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11998 0 : case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11999 : default: return 0;
12000 : }
12001 : }
12002 :
12003 : // FastEmit functions for X86ISD::VPSHUFBITQMB.
12004 :
12005 0 : unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12006 0 : if (RetVT.SimpleTy != MVT::v16i1)
12007 : return 0;
12008 0 : if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
12009 0 : return fastEmitInst_rr(X86::VPSHUFBITQMBZ128rr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12010 : }
12011 : return 0;
12012 : }
12013 :
12014 0 : unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12015 0 : if (RetVT.SimpleTy != MVT::v32i1)
12016 : return 0;
12017 0 : if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
12018 0 : return fastEmitInst_rr(X86::VPSHUFBITQMBZ256rr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12019 : }
12020 : return 0;
12021 : }
12022 :
12023 : unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12024 0 : if (RetVT.SimpleTy != MVT::v64i1)
12025 : return 0;
12026 0 : if ((Subtarget->hasBITALG())) {
12027 0 : return fastEmitInst_rr(X86::VPSHUFBITQMBZrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12028 : }
12029 : return 0;
12030 : }
12031 :
12032 0 : unsigned fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12033 0 : switch (VT.SimpleTy) {
12034 0 : case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12035 0 : case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12036 0 : case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12037 : default: return 0;
12038 : }
12039 : }
12040 :
12041 : // FastEmit functions for X86ISD::VSHL.
12042 :
12043 0 : unsigned fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12044 0 : if (RetVT.SimpleTy != MVT::v8i16)
12045 : return 0;
12046 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12047 0 : return fastEmitInst_rr(X86::VPSLLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12048 : }
12049 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12050 0 : return fastEmitInst_rr(X86::PSLLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12051 : }
12052 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12053 0 : return fastEmitInst_rr(X86::VPSLLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12054 : }
12055 : return 0;
12056 : }
12057 :
12058 0 : unsigned fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12059 0 : if (RetVT.SimpleTy != MVT::v4i32)
12060 : return 0;
12061 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12062 0 : return fastEmitInst_rr(X86::VPSLLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12063 : }
12064 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12065 0 : return fastEmitInst_rr(X86::PSLLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12066 : }
12067 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12068 0 : return fastEmitInst_rr(X86::VPSLLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12069 : }
12070 : return 0;
12071 : }
12072 :
12073 0 : unsigned fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12074 0 : if (RetVT.SimpleTy != MVT::v2i64)
12075 : return 0;
12076 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12077 0 : return fastEmitInst_rr(X86::VPSLLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12078 : }
12079 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12080 0 : return fastEmitInst_rr(X86::PSLLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12081 : }
12082 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12083 0 : return fastEmitInst_rr(X86::VPSLLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12084 : }
12085 : return 0;
12086 : }
12087 :
12088 0 : unsigned fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12089 0 : switch (VT.SimpleTy) {
12090 0 : case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12091 0 : case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12092 0 : case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12093 : default: return 0;
12094 : }
12095 : }
12096 :
12097 : // FastEmit functions for X86ISD::VSRA.
12098 :
12099 0 : unsigned fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12100 0 : if (RetVT.SimpleTy != MVT::v8i16)
12101 : return 0;
12102 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12103 0 : return fastEmitInst_rr(X86::VPSRAWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12104 : }
12105 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12106 0 : return fastEmitInst_rr(X86::PSRAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12107 : }
12108 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12109 0 : return fastEmitInst_rr(X86::VPSRAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12110 : }
12111 : return 0;
12112 : }
12113 :
12114 0 : unsigned fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12115 0 : if (RetVT.SimpleTy != MVT::v4i32)
12116 : return 0;
12117 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12118 0 : return fastEmitInst_rr(X86::VPSRADZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12119 : }
12120 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12121 0 : return fastEmitInst_rr(X86::PSRADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12122 : }
12123 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12124 0 : return fastEmitInst_rr(X86::VPSRADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12125 : }
12126 : return 0;
12127 : }
12128 :
12129 0 : unsigned fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12130 0 : if (RetVT.SimpleTy != MVT::v2i64)
12131 : return 0;
12132 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12133 0 : return fastEmitInst_rr(X86::VPSRAQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12134 : }
12135 : return 0;
12136 : }
12137 :
12138 0 : unsigned fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12139 0 : switch (VT.SimpleTy) {
12140 0 : case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12141 0 : case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12142 0 : case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12143 : default: return 0;
12144 : }
12145 : }
12146 :
12147 : // FastEmit functions for X86ISD::VSRAV.
12148 :
12149 0 : unsigned fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12150 0 : if (RetVT.SimpleTy != MVT::v8i16)
12151 : return 0;
12152 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12153 0 : return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12154 : }
12155 : return 0;
12156 : }
12157 :
12158 0 : unsigned fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12159 0 : if (RetVT.SimpleTy != MVT::v16i16)
12160 : return 0;
12161 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12162 0 : return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12163 : }
12164 : return 0;
12165 : }
12166 :
12167 : unsigned fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12168 0 : if (RetVT.SimpleTy != MVT::v32i16)
12169 : return 0;
12170 0 : if ((Subtarget->hasBWI())) {
12171 0 : return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12172 : }
12173 : return 0;
12174 : }
12175 :
12176 0 : unsigned fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12177 0 : if (RetVT.SimpleTy != MVT::v4i32)
12178 : return 0;
12179 0 : if ((Subtarget->hasVLX())) {
12180 0 : return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12181 : }
12182 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12183 0 : return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12184 : }
12185 : return 0;
12186 : }
12187 :
12188 0 : unsigned fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12189 0 : if (RetVT.SimpleTy != MVT::v8i32)
12190 : return 0;
12191 0 : if ((Subtarget->hasVLX())) {
12192 0 : return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12193 : }
12194 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12195 0 : return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12196 : }
12197 : return 0;
12198 : }
12199 :
12200 : unsigned fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12201 0 : if (RetVT.SimpleTy != MVT::v16i32)
12202 : return 0;
12203 0 : if ((Subtarget->hasAVX512())) {
12204 0 : return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12205 : }
12206 : return 0;
12207 : }
12208 :
12209 : unsigned fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12210 0 : if (RetVT.SimpleTy != MVT::v2i64)
12211 : return 0;
12212 0 : if ((Subtarget->hasVLX())) {
12213 0 : return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12214 : }
12215 : return 0;
12216 : }
12217 :
12218 : unsigned fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12219 0 : if (RetVT.SimpleTy != MVT::v4i64)
12220 : return 0;
12221 0 : if ((Subtarget->hasVLX())) {
12222 0 : return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12223 : }
12224 : return 0;
12225 : }
12226 :
12227 : unsigned fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12228 0 : if (RetVT.SimpleTy != MVT::v8i64)
12229 : return 0;
12230 0 : if ((Subtarget->hasAVX512())) {
12231 0 : return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12232 : }
12233 : return 0;
12234 : }
12235 :
12236 0 : unsigned fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12237 0 : switch (VT.SimpleTy) {
12238 0 : case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12239 0 : case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12240 0 : case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12241 0 : case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12242 0 : case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12243 0 : case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12244 0 : case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12245 0 : case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12246 0 : case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12247 : default: return 0;
12248 : }
12249 : }
12250 :
12251 : // FastEmit functions for X86ISD::VSRL.
12252 :
12253 0 : unsigned fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12254 0 : if (RetVT.SimpleTy != MVT::v8i16)
12255 : return 0;
12256 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12257 0 : return fastEmitInst_rr(X86::VPSRLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12258 : }
12259 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12260 0 : return fastEmitInst_rr(X86::PSRLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12261 : }
12262 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12263 0 : return fastEmitInst_rr(X86::VPSRLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12264 : }
12265 : return 0;
12266 : }
12267 :
12268 0 : unsigned fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12269 0 : if (RetVT.SimpleTy != MVT::v4i32)
12270 : return 0;
12271 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12272 0 : return fastEmitInst_rr(X86::VPSRLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12273 : }
12274 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12275 0 : return fastEmitInst_rr(X86::PSRLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12276 : }
12277 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12278 0 : return fastEmitInst_rr(X86::VPSRLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12279 : }
12280 : return 0;
12281 : }
12282 :
12283 0 : unsigned fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12284 0 : if (RetVT.SimpleTy != MVT::v2i64)
12285 : return 0;
12286 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
12287 0 : return fastEmitInst_rr(X86::VPSRLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12288 : }
12289 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12290 0 : return fastEmitInst_rr(X86::PSRLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12291 : }
12292 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12293 0 : return fastEmitInst_rr(X86::VPSRLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12294 : }
12295 : return 0;
12296 : }
12297 :
12298 0 : unsigned fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12299 0 : switch (VT.SimpleTy) {
12300 0 : case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12301 0 : case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12302 0 : case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12303 : default: return 0;
12304 : }
12305 : }
12306 :
12307 : // Top-level FastEmit function.
12308 :
12309 247831 : unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) override {
12310 247831 : switch (Opcode) {
12311 83656 : case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12312 30951 : case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12313 1868 : case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12314 2310 : case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12315 3536 : case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12316 2028 : case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12317 1323 : case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12318 0 : case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12319 0 : case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12320 7990 : case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12321 0 : case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12322 0 : case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12323 3169 : case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12324 0 : case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12325 0 : case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12326 18 : case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12327 3607 : case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12328 76374 : case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12329 0 : case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12330 0 : case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12331 1593 : case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12332 0 : case X86ISD::ADDS: return fastEmit_X86ISD_ADDS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12333 0 : case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12334 0 : case X86ISD::ADDUS: return fastEmit_X86ISD_ADDUS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12335 0 : case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12336 0 : case X86ISD::AVG: return fastEmit_X86ISD_AVG_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12337 0 : case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12338 0 : case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12339 0 : case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12340 0 : case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12341 0 : case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12342 0 : case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12343 0 : case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12344 0 : case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12345 0 : case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12346 0 : case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12347 0 : case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12348 0 : case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12349 0 : case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12350 0 : case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12351 0 : case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12352 0 : case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12353 0 : case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12354 0 : case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12355 0 : case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12356 0 : case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12357 0 : case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12358 0 : case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12359 0 : case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12360 0 : case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12361 0 : case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12362 0 : case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12363 0 : case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12364 0 : case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12365 0 : case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12366 0 : case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12367 0 : case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12368 0 : case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12369 0 : case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12370 0 : case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12371 0 : case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12372 0 : case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12373 0 : case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12374 0 : case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12375 0 : case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12376 0 : case X86ISD::SUBS: return fastEmit_X86ISD_SUBS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12377 0 : case X86ISD::SUBUS: return fastEmit_X86ISD_SUBUS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12378 0 : case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12379 0 : case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12380 0 : case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12381 0 : case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12382 0 : case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12383 0 : case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12384 0 : case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12385 0 : case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12386 0 : case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12387 0 : case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12388 0 : case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12389 0 : case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12390 0 : case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12391 0 : case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12392 : default: return 0;
12393 : }
12394 : }
12395 :
12396 : // FastEmit functions for ISD::ADD.
12397 :
12398 : unsigned fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12399 24 : if (RetVT.SimpleTy != MVT::i8)
12400 : return 0;
12401 24 : return fastEmitInst_ri(X86::ADD8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12402 : }
12403 :
12404 : unsigned fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12405 0 : if (RetVT.SimpleTy != MVT::i16)
12406 : return 0;
12407 0 : return fastEmitInst_ri(X86::ADD16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12408 : }
12409 :
12410 : unsigned fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12411 72 : if (RetVT.SimpleTy != MVT::i32)
12412 : return 0;
12413 72 : return fastEmitInst_ri(X86::ADD32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12414 : }
12415 :
12416 122 : unsigned fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12417 122 : switch (VT.SimpleTy) {
12418 24 : case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12419 0 : case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12420 72 : case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12421 : default: return 0;
12422 : }
12423 : }
12424 :
12425 : // FastEmit functions for ISD::AND.
12426 :
12427 : unsigned fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12428 41534 : if (RetVT.SimpleTy != MVT::i8)
12429 : return 0;
12430 41534 : return fastEmitInst_ri(X86::AND8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12431 : }
12432 :
12433 : unsigned fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12434 6 : if (RetVT.SimpleTy != MVT::i16)
12435 : return 0;
12436 6 : return fastEmitInst_ri(X86::AND16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12437 : }
12438 :
12439 : unsigned fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12440 1547 : if (RetVT.SimpleTy != MVT::i32)
12441 : return 0;
12442 1547 : return fastEmitInst_ri(X86::AND32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12443 : }
12444 :
12445 44018 : unsigned fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12446 44018 : switch (VT.SimpleTy) {
12447 41534 : case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12448 6 : case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12449 1547 : case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12450 : default: return 0;
12451 : }
12452 : }
12453 :
12454 : // FastEmit functions for ISD::EXTRACT_SUBVECTOR.
12455 :
12456 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v32i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12457 0 : if (RetVT.SimpleTy != MVT::v16i8)
12458 : return 0;
12459 0 : if ((Subtarget->hasVLX())) {
12460 0 : return fastEmitInst_ri(X86::VEXTRACTI32x4Z256rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12461 : }
12462 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12463 0 : return fastEmitInst_ri(X86::VEXTRACTI128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12464 : }
12465 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12466 0 : return fastEmitInst_ri(X86::VEXTRACTF128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12467 : }
12468 : return 0;
12469 : }
12470 :
12471 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v64i8_MVT_v16i8_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12472 0 : if ((Subtarget->hasAVX512())) {
12473 0 : return fastEmitInst_ri(X86::VEXTRACTI32x4Zrr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12474 : }
12475 : return 0;
12476 : }
12477 :
12478 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v64i8_MVT_v32i8_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12479 0 : if ((Subtarget->hasAVX512())) {
12480 0 : return fastEmitInst_ri(X86::VEXTRACTI64x4Zrr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
12481 : }
12482 : return 0;
12483 : }
12484 :
12485 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v64i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12486 0 : switch (RetVT.SimpleTy) {
12487 0 : case MVT::v16i8: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v64i8_MVT_v16i8_ri(Op0, Op0IsKill, imm1);
12488 0 : case MVT::v32i8: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v64i8_MVT_v32i8_ri(Op0, Op0IsKill, imm1);
12489 : default: return 0;
12490 : }
12491 : }
12492 :
12493 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12494 0 : if (RetVT.SimpleTy != MVT::v8i16)
12495 : return 0;
12496 0 : if ((Subtarget->hasVLX())) {
12497 0 : return fastEmitInst_ri(X86::VEXTRACTI32x4Z256rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12498 : }
12499 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12500 0 : return fastEmitInst_ri(X86::VEXTRACTI128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12501 : }
12502 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12503 0 : return fastEmitInst_ri(X86::VEXTRACTF128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12504 : }
12505 : return 0;
12506 : }
12507 :
12508 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v32i16_MVT_v8i16_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12509 0 : if ((Subtarget->hasAVX512())) {
12510 0 : return fastEmitInst_ri(X86::VEXTRACTI32x4Zrr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12511 : }
12512 : return 0;
12513 : }
12514 :
12515 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v32i16_MVT_v16i16_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12516 0 : if ((Subtarget->hasAVX512())) {
12517 0 : return fastEmitInst_ri(X86::VEXTRACTI64x4Zrr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
12518 : }
12519 : return 0;
12520 : }
12521 :
12522 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v32i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12523 0 : switch (RetVT.SimpleTy) {
12524 0 : case MVT::v8i16: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v32i16_MVT_v8i16_ri(Op0, Op0IsKill, imm1);
12525 0 : case MVT::v16i16: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v32i16_MVT_v16i16_ri(Op0, Op0IsKill, imm1);
12526 : default: return 0;
12527 : }
12528 : }
12529 :
12530 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12531 0 : if (RetVT.SimpleTy != MVT::v4i32)
12532 : return 0;
12533 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12534 0 : return fastEmitInst_ri(X86::VEXTRACTI128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12535 : }
12536 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12537 0 : return fastEmitInst_ri(X86::VEXTRACTF128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12538 : }
12539 0 : if ((Subtarget->hasVLX())) {
12540 0 : return fastEmitInst_ri(X86::VEXTRACTI32x4Z256rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12541 : }
12542 : return 0;
12543 : }
12544 :
12545 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16i32_MVT_v4i32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12546 0 : if ((Subtarget->hasAVX512())) {
12547 0 : return fastEmitInst_ri(X86::VEXTRACTI32x4Zrr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12548 : }
12549 : return 0;
12550 : }
12551 :
12552 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16i32_MVT_v8i32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12553 0 : if ((Subtarget->hasAVX512())) {
12554 0 : return fastEmitInst_ri(X86::VEXTRACTI64x4Zrr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
12555 : }
12556 : return 0;
12557 : }
12558 :
12559 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12560 0 : switch (RetVT.SimpleTy) {
12561 0 : case MVT::v4i32: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16i32_MVT_v4i32_ri(Op0, Op0IsKill, imm1);
12562 0 : case MVT::v8i32: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16i32_MVT_v8i32_ri(Op0, Op0IsKill, imm1);
12563 : default: return 0;
12564 : }
12565 : }
12566 :
12567 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v4i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12568 0 : if (RetVT.SimpleTy != MVT::v2i64)
12569 : return 0;
12570 0 : if ((Subtarget->hasVLX())) {
12571 0 : return fastEmitInst_ri(X86::VEXTRACTI32x4Z256rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12572 : }
12573 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
12574 0 : return fastEmitInst_ri(X86::VEXTRACTI128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12575 : }
12576 0 : if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
12577 0 : return fastEmitInst_ri(X86::VEXTRACTF128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12578 : }
12579 : return 0;
12580 : }
12581 :
12582 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8i64_MVT_v2i64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12583 0 : if ((Subtarget->hasAVX512())) {
12584 0 : return fastEmitInst_ri(X86::VEXTRACTI32x4Zrr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12585 : }
12586 : return 0;
12587 : }
12588 :
12589 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8i64_MVT_v4i64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12590 0 : if ((Subtarget->hasAVX512())) {
12591 0 : return fastEmitInst_ri(X86::VEXTRACTI64x4Zrr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
12592 : }
12593 : return 0;
12594 : }
12595 :
12596 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12597 0 : switch (RetVT.SimpleTy) {
12598 0 : case MVT::v2i64: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8i64_MVT_v2i64_ri(Op0, Op0IsKill, imm1);
12599 0 : case MVT::v4i64: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8i64_MVT_v4i64_ri(Op0, Op0IsKill, imm1);
12600 : default: return 0;
12601 : }
12602 : }
12603 :
12604 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12605 0 : if (RetVT.SimpleTy != MVT::v4f32)
12606 : return 0;
12607 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12608 0 : return fastEmitInst_ri(X86::VEXTRACTF128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12609 : }
12610 0 : if ((Subtarget->hasVLX())) {
12611 0 : return fastEmitInst_ri(X86::VEXTRACTF32x4Z256rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12612 : }
12613 : return 0;
12614 : }
12615 :
12616 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16f32_MVT_v4f32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12617 0 : if ((Subtarget->hasAVX512())) {
12618 0 : return fastEmitInst_ri(X86::VEXTRACTF32x4Zrr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12619 : }
12620 : return 0;
12621 : }
12622 :
12623 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16f32_MVT_v8f32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12624 0 : if ((Subtarget->hasAVX512())) {
12625 0 : return fastEmitInst_ri(X86::VEXTRACTF64x4Zrr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
12626 : }
12627 : return 0;
12628 : }
12629 :
12630 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12631 0 : switch (RetVT.SimpleTy) {
12632 0 : case MVT::v4f32: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16f32_MVT_v4f32_ri(Op0, Op0IsKill, imm1);
12633 0 : case MVT::v8f32: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16f32_MVT_v8f32_ri(Op0, Op0IsKill, imm1);
12634 : default: return 0;
12635 : }
12636 : }
12637 :
12638 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v4f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12639 0 : if (RetVT.SimpleTy != MVT::v2f64)
12640 : return 0;
12641 0 : if ((Subtarget->hasVLX())) {
12642 0 : return fastEmitInst_ri(X86::VEXTRACTF32x4Z256rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12643 : }
12644 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12645 0 : return fastEmitInst_ri(X86::VEXTRACTF128rr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
12646 : }
12647 : return 0;
12648 : }
12649 :
12650 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8f64_MVT_v2f64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12651 0 : if ((Subtarget->hasAVX512())) {
12652 0 : return fastEmitInst_ri(X86::VEXTRACTF32x4Zrr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
12653 : }
12654 : return 0;
12655 : }
12656 :
12657 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8f64_MVT_v4f64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12658 0 : if ((Subtarget->hasAVX512())) {
12659 0 : return fastEmitInst_ri(X86::VEXTRACTF64x4Zrr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
12660 : }
12661 : return 0;
12662 : }
12663 :
12664 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12665 0 : switch (RetVT.SimpleTy) {
12666 0 : case MVT::v2f64: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8f64_MVT_v2f64_ri(Op0, Op0IsKill, imm1);
12667 0 : case MVT::v4f64: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8f64_MVT_v4f64_ri(Op0, Op0IsKill, imm1);
12668 : default: return 0;
12669 : }
12670 : }
12671 :
12672 0 : unsigned fastEmit_ISD_EXTRACT_SUBVECTOR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12673 0 : switch (VT.SimpleTy) {
12674 0 : case MVT::v32i8: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v32i8_ri(RetVT, Op0, Op0IsKill, imm1);
12675 0 : case MVT::v64i8: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v64i8_ri(RetVT, Op0, Op0IsKill, imm1);
12676 0 : case MVT::v16i16: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16i16_ri(RetVT, Op0, Op0IsKill, imm1);
12677 0 : case MVT::v32i16: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v32i16_ri(RetVT, Op0, Op0IsKill, imm1);
12678 0 : case MVT::v8i32: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8i32_ri(RetVT, Op0, Op0IsKill, imm1);
12679 0 : case MVT::v16i32: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
12680 0 : case MVT::v4i64: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v4i64_ri(RetVT, Op0, Op0IsKill, imm1);
12681 0 : case MVT::v8i64: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
12682 0 : case MVT::v8f32: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
12683 0 : case MVT::v16f32: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
12684 0 : case MVT::v4f64: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v4f64_ri(RetVT, Op0, Op0IsKill, imm1);
12685 0 : case MVT::v8f64: return fastEmit_ISD_EXTRACT_SUBVECTOR_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
12686 : default: return 0;
12687 : }
12688 : }
12689 :
12690 : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
12691 :
12692 0 : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12693 0 : if (RetVT.SimpleTy != MVT::i32)
12694 : return 0;
12695 0 : if ((Subtarget->hasDQI())) {
12696 0 : return fastEmitInst_ri(X86::VPEXTRDZrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12697 : }
12698 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12699 0 : return fastEmitInst_ri(X86::PEXTRDrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12700 : }
12701 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
12702 0 : return fastEmitInst_ri(X86::VPEXTRDrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12703 : }
12704 : return 0;
12705 : }
12706 :
12707 0 : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12708 0 : if (RetVT.SimpleTy != MVT::i64)
12709 : return 0;
12710 0 : if ((Subtarget->hasDQI())) {
12711 0 : return fastEmitInst_ri(X86::VPEXTRQZrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
12712 : }
12713 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12714 0 : return fastEmitInst_ri(X86::PEXTRQrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
12715 : }
12716 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
12717 0 : return fastEmitInst_ri(X86::VPEXTRQrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
12718 : }
12719 : return 0;
12720 : }
12721 :
12722 0 : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12723 0 : switch (VT.SimpleTy) {
12724 0 : case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
12725 0 : case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
12726 : default: return 0;
12727 : }
12728 : }
12729 :
12730 : // FastEmit functions for ISD::MUL.
12731 :
12732 : unsigned fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12733 0 : if (RetVT.SimpleTy != MVT::i16)
12734 : return 0;
12735 0 : return fastEmitInst_ri(X86::IMUL16rri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12736 : }
12737 :
12738 : unsigned fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12739 552 : if (RetVT.SimpleTy != MVT::i32)
12740 : return 0;
12741 552 : return fastEmitInst_ri(X86::IMUL32rri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12742 : }
12743 :
12744 1122 : unsigned fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12745 1122 : switch (VT.SimpleTy) {
12746 0 : case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12747 552 : case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12748 : default: return 0;
12749 : }
12750 : }
12751 :
12752 : // FastEmit functions for ISD::OR.
12753 :
12754 : unsigned fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12755 6 : if (RetVT.SimpleTy != MVT::i8)
12756 : return 0;
12757 6 : return fastEmitInst_ri(X86::OR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12758 : }
12759 :
12760 : unsigned fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12761 0 : if (RetVT.SimpleTy != MVT::i16)
12762 : return 0;
12763 0 : return fastEmitInst_ri(X86::OR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12764 : }
12765 :
12766 : unsigned fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12767 15 : if (RetVT.SimpleTy != MVT::i32)
12768 : return 0;
12769 15 : return fastEmitInst_ri(X86::OR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12770 : }
12771 :
12772 21 : unsigned fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12773 21 : switch (VT.SimpleTy) {
12774 6 : case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12775 0 : case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12776 15 : case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12777 : default: return 0;
12778 : }
12779 : }
12780 :
12781 : // FastEmit functions for ISD::ROTL.
12782 :
12783 : unsigned fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12784 0 : if (RetVT.SimpleTy != MVT::i8)
12785 : return 0;
12786 0 : return fastEmitInst_ri(X86::ROL8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12787 : }
12788 :
12789 : unsigned fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12790 0 : if (RetVT.SimpleTy != MVT::i16)
12791 : return 0;
12792 0 : return fastEmitInst_ri(X86::ROL16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12793 : }
12794 :
12795 : unsigned fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12796 0 : if (RetVT.SimpleTy != MVT::i32)
12797 : return 0;
12798 0 : return fastEmitInst_ri(X86::ROL32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12799 : }
12800 :
12801 : unsigned fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12802 0 : if (RetVT.SimpleTy != MVT::i64)
12803 : return 0;
12804 0 : return fastEmitInst_ri(X86::ROL64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
12805 : }
12806 :
12807 0 : unsigned fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12808 0 : switch (VT.SimpleTy) {
12809 0 : case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12810 0 : case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12811 0 : case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12812 0 : case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
12813 : default: return 0;
12814 : }
12815 : }
12816 :
12817 : // FastEmit functions for ISD::SHL.
12818 :
12819 : unsigned fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12820 2 : if (RetVT.SimpleTy != MVT::i8)
12821 : return 0;
12822 2 : return fastEmitInst_ri(X86::SHL8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12823 : }
12824 :
12825 : unsigned fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12826 4 : if (RetVT.SimpleTy != MVT::i16)
12827 : return 0;
12828 4 : return fastEmitInst_ri(X86::SHL16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12829 : }
12830 :
12831 : unsigned fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12832 4893 : if (RetVT.SimpleTy != MVT::i32)
12833 : return 0;
12834 4893 : return fastEmitInst_ri(X86::SHL32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12835 : }
12836 :
12837 : unsigned fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12838 55189 : if (RetVT.SimpleTy != MVT::i64)
12839 : return 0;
12840 55189 : return fastEmitInst_ri(X86::SHL64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
12841 : }
12842 :
12843 60088 : unsigned fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12844 60088 : switch (VT.SimpleTy) {
12845 2 : case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12846 4 : case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12847 4893 : case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12848 55189 : case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
12849 : default: return 0;
12850 : }
12851 : }
12852 :
12853 : // FastEmit functions for ISD::SRA.
12854 :
12855 : unsigned fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12856 2 : if (RetVT.SimpleTy != MVT::i8)
12857 : return 0;
12858 2 : return fastEmitInst_ri(X86::SAR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12859 : }
12860 :
12861 : unsigned fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12862 2 : if (RetVT.SimpleTy != MVT::i16)
12863 : return 0;
12864 2 : return fastEmitInst_ri(X86::SAR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12865 : }
12866 :
12867 : unsigned fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12868 18825 : if (RetVT.SimpleTy != MVT::i32)
12869 : return 0;
12870 18825 : return fastEmitInst_ri(X86::SAR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12871 : }
12872 :
12873 : unsigned fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12874 33247 : if (RetVT.SimpleTy != MVT::i64)
12875 : return 0;
12876 33247 : return fastEmitInst_ri(X86::SAR64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
12877 : }
12878 :
12879 52076 : unsigned fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12880 52076 : switch (VT.SimpleTy) {
12881 2 : case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12882 2 : case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12883 18825 : case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12884 33247 : case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
12885 : default: return 0;
12886 : }
12887 : }
12888 :
12889 : // FastEmit functions for ISD::SRL.
12890 :
12891 : unsigned fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12892 2 : if (RetVT.SimpleTy != MVT::i8)
12893 : return 0;
12894 2 : return fastEmitInst_ri(X86::SHR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12895 : }
12896 :
12897 : unsigned fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12898 3 : if (RetVT.SimpleTy != MVT::i16)
12899 : return 0;
12900 3 : return fastEmitInst_ri(X86::SHR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12901 : }
12902 :
12903 : unsigned fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12904 463 : if (RetVT.SimpleTy != MVT::i32)
12905 : return 0;
12906 463 : return fastEmitInst_ri(X86::SHR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12907 : }
12908 :
12909 : unsigned fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12910 5144 : if (RetVT.SimpleTy != MVT::i64)
12911 : return 0;
12912 5144 : return fastEmitInst_ri(X86::SHR64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
12913 : }
12914 :
12915 5612 : unsigned fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12916 5612 : switch (VT.SimpleTy) {
12917 2 : case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12918 3 : case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12919 463 : case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12920 5144 : case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
12921 : default: return 0;
12922 : }
12923 : }
12924 :
12925 : // FastEmit functions for ISD::SUB.
12926 :
12927 : unsigned fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12928 0 : if (RetVT.SimpleTy != MVT::i8)
12929 : return 0;
12930 0 : return fastEmitInst_ri(X86::SUB8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12931 : }
12932 :
12933 : unsigned fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12934 0 : if (RetVT.SimpleTy != MVT::i16)
12935 : return 0;
12936 0 : return fastEmitInst_ri(X86::SUB16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12937 : }
12938 :
12939 : unsigned fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12940 44 : if (RetVT.SimpleTy != MVT::i32)
12941 : return 0;
12942 44 : return fastEmitInst_ri(X86::SUB32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12943 : }
12944 :
12945 44 : unsigned fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12946 44 : switch (VT.SimpleTy) {
12947 0 : case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12948 0 : case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12949 44 : case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12950 : default: return 0;
12951 : }
12952 : }
12953 :
12954 : // FastEmit functions for ISD::XOR.
12955 :
12956 : unsigned fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12957 21813 : if (RetVT.SimpleTy != MVT::i8)
12958 : return 0;
12959 21813 : return fastEmitInst_ri(X86::XOR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
12960 : }
12961 :
12962 : unsigned fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12963 0 : if (RetVT.SimpleTy != MVT::i16)
12964 : return 0;
12965 0 : return fastEmitInst_ri(X86::XOR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
12966 : }
12967 :
12968 : unsigned fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12969 12 : if (RetVT.SimpleTy != MVT::i32)
12970 : return 0;
12971 12 : return fastEmitInst_ri(X86::XOR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12972 : }
12973 :
12974 21982 : unsigned fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12975 21982 : switch (VT.SimpleTy) {
12976 21813 : case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
12977 0 : case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
12978 12 : case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12979 : default: return 0;
12980 : }
12981 : }
12982 :
12983 : // FastEmit functions for X86ISD::BEXTR.
12984 :
12985 : unsigned fastEmit_X86ISD_BEXTR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12986 0 : if (RetVT.SimpleTy != MVT::i32)
12987 : return 0;
12988 0 : if ((Subtarget->hasTBM())) {
12989 0 : return fastEmitInst_ri(X86::BEXTRI32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
12990 : }
12991 : return 0;
12992 : }
12993 :
12994 0 : unsigned fastEmit_X86ISD_BEXTR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
12995 0 : switch (VT.SimpleTy) {
12996 0 : case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
12997 : default: return 0;
12998 : }
12999 : }
13000 :
13001 : // FastEmit functions for X86ISD::CVTP2SI_RND.
13002 :
13003 : unsigned fastEmit_X86ISD_CVTP2SI_RND_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13004 0 : if (RetVT.SimpleTy != MVT::v8i64)
13005 : return 0;
13006 0 : if ((Subtarget->hasDQI())) {
13007 0 : return fastEmitInst_ri(X86::VCVTPS2QQZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13008 : }
13009 : return 0;
13010 : }
13011 :
13012 : unsigned fastEmit_X86ISD_CVTP2SI_RND_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13013 0 : if (RetVT.SimpleTy != MVT::v16i32)
13014 : return 0;
13015 0 : if ((Subtarget->hasAVX512())) {
13016 0 : return fastEmitInst_ri(X86::VCVTPS2DQZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13017 : }
13018 : return 0;
13019 : }
13020 :
13021 : unsigned fastEmit_X86ISD_CVTP2SI_RND_MVT_v8f64_MVT_v8i32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13022 0 : if ((Subtarget->hasAVX512())) {
13023 0 : return fastEmitInst_ri(X86::VCVTPD2DQZrrb, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13024 : }
13025 : return 0;
13026 : }
13027 :
13028 : unsigned fastEmit_X86ISD_CVTP2SI_RND_MVT_v8f64_MVT_v8i64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13029 0 : if ((Subtarget->hasDQI())) {
13030 0 : return fastEmitInst_ri(X86::VCVTPD2QQZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13031 : }
13032 : return 0;
13033 : }
13034 :
13035 0 : unsigned fastEmit_X86ISD_CVTP2SI_RND_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13036 0 : switch (RetVT.SimpleTy) {
13037 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_RND_MVT_v8f64_MVT_v8i32_ri(Op0, Op0IsKill, imm1);
13038 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_RND_MVT_v8f64_MVT_v8i64_ri(Op0, Op0IsKill, imm1);
13039 : default: return 0;
13040 : }
13041 : }
13042 :
13043 0 : unsigned fastEmit_X86ISD_CVTP2SI_RND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13044 0 : switch (VT.SimpleTy) {
13045 0 : case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_RND_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
13046 0 : case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_RND_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
13047 0 : case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_RND_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
13048 : default: return 0;
13049 : }
13050 : }
13051 :
13052 : // FastEmit functions for X86ISD::CVTP2UI_RND.
13053 :
13054 : unsigned fastEmit_X86ISD_CVTP2UI_RND_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13055 0 : if (RetVT.SimpleTy != MVT::v8i64)
13056 : return 0;
13057 0 : if ((Subtarget->hasDQI())) {
13058 0 : return fastEmitInst_ri(X86::VCVTPS2UQQZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13059 : }
13060 : return 0;
13061 : }
13062 :
13063 : unsigned fastEmit_X86ISD_CVTP2UI_RND_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13064 0 : if (RetVT.SimpleTy != MVT::v16i32)
13065 : return 0;
13066 0 : if ((Subtarget->hasAVX512())) {
13067 0 : return fastEmitInst_ri(X86::VCVTPS2UDQZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13068 : }
13069 : return 0;
13070 : }
13071 :
13072 : unsigned fastEmit_X86ISD_CVTP2UI_RND_MVT_v8f64_MVT_v8i32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13073 0 : if ((Subtarget->hasAVX512())) {
13074 0 : return fastEmitInst_ri(X86::VCVTPD2UDQZrrb, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13075 : }
13076 : return 0;
13077 : }
13078 :
13079 : unsigned fastEmit_X86ISD_CVTP2UI_RND_MVT_v8f64_MVT_v8i64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13080 0 : if ((Subtarget->hasDQI())) {
13081 0 : return fastEmitInst_ri(X86::VCVTPD2UQQZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13082 : }
13083 : return 0;
13084 : }
13085 :
13086 0 : unsigned fastEmit_X86ISD_CVTP2UI_RND_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13087 0 : switch (RetVT.SimpleTy) {
13088 0 : case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_RND_MVT_v8f64_MVT_v8i32_ri(Op0, Op0IsKill, imm1);
13089 0 : case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_RND_MVT_v8f64_MVT_v8i64_ri(Op0, Op0IsKill, imm1);
13090 : default: return 0;
13091 : }
13092 : }
13093 :
13094 0 : unsigned fastEmit_X86ISD_CVTP2UI_RND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13095 0 : switch (VT.SimpleTy) {
13096 0 : case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_RND_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
13097 0 : case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_RND_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
13098 0 : case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_RND_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
13099 : default: return 0;
13100 : }
13101 : }
13102 :
13103 : // FastEmit functions for X86ISD::CVTPS2PH.
13104 :
13105 0 : unsigned fastEmit_X86ISD_CVTPS2PH_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13106 0 : if (RetVT.SimpleTy != MVT::v8i16)
13107 : return 0;
13108 0 : if ((Subtarget->hasVLX())) {
13109 0 : return fastEmitInst_ri(X86::VCVTPS2PHZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13110 : }
13111 0 : if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
13112 0 : return fastEmitInst_ri(X86::VCVTPS2PHrr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13113 : }
13114 : return 0;
13115 : }
13116 :
13117 0 : unsigned fastEmit_X86ISD_CVTPS2PH_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13118 0 : if (RetVT.SimpleTy != MVT::v8i16)
13119 : return 0;
13120 0 : if ((Subtarget->hasVLX())) {
13121 0 : return fastEmitInst_ri(X86::VCVTPS2PHZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13122 : }
13123 0 : if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
13124 0 : return fastEmitInst_ri(X86::VCVTPS2PHYrr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13125 : }
13126 : return 0;
13127 : }
13128 :
13129 : unsigned fastEmit_X86ISD_CVTPS2PH_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13130 0 : if (RetVT.SimpleTy != MVT::v16i16)
13131 : return 0;
13132 0 : if ((Subtarget->hasAVX512())) {
13133 0 : return fastEmitInst_ri(X86::VCVTPS2PHZrr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13134 : }
13135 : return 0;
13136 : }
13137 :
13138 0 : unsigned fastEmit_X86ISD_CVTPS2PH_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13139 0 : switch (VT.SimpleTy) {
13140 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTPS2PH_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
13141 0 : case MVT::v8f32: return fastEmit_X86ISD_CVTPS2PH_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
13142 0 : case MVT::v16f32: return fastEmit_X86ISD_CVTPS2PH_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
13143 : default: return 0;
13144 : }
13145 : }
13146 :
13147 : // FastEmit functions for X86ISD::CVTS2SI_RND.
13148 :
13149 : unsigned fastEmit_X86ISD_CVTS2SI_RND_MVT_v4f32_MVT_i32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13150 0 : if ((Subtarget->hasAVX512())) {
13151 0 : return fastEmitInst_ri(X86::VCVTSS2SIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13152 : }
13153 : return 0;
13154 : }
13155 :
13156 : unsigned fastEmit_X86ISD_CVTS2SI_RND_MVT_v4f32_MVT_i64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13157 0 : if ((Subtarget->hasAVX512())) {
13158 0 : return fastEmitInst_ri(X86::VCVTSS2SI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
13159 : }
13160 : return 0;
13161 : }
13162 :
13163 0 : unsigned fastEmit_X86ISD_CVTS2SI_RND_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13164 0 : switch (RetVT.SimpleTy) {
13165 0 : case MVT::i32: return fastEmit_X86ISD_CVTS2SI_RND_MVT_v4f32_MVT_i32_ri(Op0, Op0IsKill, imm1);
13166 0 : case MVT::i64: return fastEmit_X86ISD_CVTS2SI_RND_MVT_v4f32_MVT_i64_ri(Op0, Op0IsKill, imm1);
13167 : default: return 0;
13168 : }
13169 : }
13170 :
13171 : unsigned fastEmit_X86ISD_CVTS2SI_RND_MVT_v2f64_MVT_i32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13172 0 : if ((Subtarget->hasAVX512())) {
13173 0 : return fastEmitInst_ri(X86::VCVTSD2SIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13174 : }
13175 : return 0;
13176 : }
13177 :
13178 : unsigned fastEmit_X86ISD_CVTS2SI_RND_MVT_v2f64_MVT_i64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13179 0 : if ((Subtarget->hasAVX512())) {
13180 0 : return fastEmitInst_ri(X86::VCVTSD2SI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
13181 : }
13182 : return 0;
13183 : }
13184 :
13185 0 : unsigned fastEmit_X86ISD_CVTS2SI_RND_MVT_v2f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13186 0 : switch (RetVT.SimpleTy) {
13187 0 : case MVT::i32: return fastEmit_X86ISD_CVTS2SI_RND_MVT_v2f64_MVT_i32_ri(Op0, Op0IsKill, imm1);
13188 0 : case MVT::i64: return fastEmit_X86ISD_CVTS2SI_RND_MVT_v2f64_MVT_i64_ri(Op0, Op0IsKill, imm1);
13189 : default: return 0;
13190 : }
13191 : }
13192 :
13193 0 : unsigned fastEmit_X86ISD_CVTS2SI_RND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13194 0 : switch (VT.SimpleTy) {
13195 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_RND_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
13196 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_RND_MVT_v2f64_ri(RetVT, Op0, Op0IsKill, imm1);
13197 : default: return 0;
13198 : }
13199 : }
13200 :
13201 : // FastEmit functions for X86ISD::CVTS2UI_RND.
13202 :
13203 : unsigned fastEmit_X86ISD_CVTS2UI_RND_MVT_v4f32_MVT_i32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13204 0 : if ((Subtarget->hasAVX512())) {
13205 0 : return fastEmitInst_ri(X86::VCVTSS2USIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13206 : }
13207 : return 0;
13208 : }
13209 :
13210 : unsigned fastEmit_X86ISD_CVTS2UI_RND_MVT_v4f32_MVT_i64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13211 0 : if ((Subtarget->hasAVX512())) {
13212 0 : return fastEmitInst_ri(X86::VCVTSS2USI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
13213 : }
13214 : return 0;
13215 : }
13216 :
13217 0 : unsigned fastEmit_X86ISD_CVTS2UI_RND_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13218 0 : switch (RetVT.SimpleTy) {
13219 0 : case MVT::i32: return fastEmit_X86ISD_CVTS2UI_RND_MVT_v4f32_MVT_i32_ri(Op0, Op0IsKill, imm1);
13220 0 : case MVT::i64: return fastEmit_X86ISD_CVTS2UI_RND_MVT_v4f32_MVT_i64_ri(Op0, Op0IsKill, imm1);
13221 : default: return 0;
13222 : }
13223 : }
13224 :
13225 : unsigned fastEmit_X86ISD_CVTS2UI_RND_MVT_v2f64_MVT_i32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13226 0 : if ((Subtarget->hasAVX512())) {
13227 0 : return fastEmitInst_ri(X86::VCVTSD2USIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13228 : }
13229 : return 0;
13230 : }
13231 :
13232 : unsigned fastEmit_X86ISD_CVTS2UI_RND_MVT_v2f64_MVT_i64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13233 0 : if ((Subtarget->hasAVX512())) {
13234 0 : return fastEmitInst_ri(X86::VCVTSD2USI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
13235 : }
13236 : return 0;
13237 : }
13238 :
13239 0 : unsigned fastEmit_X86ISD_CVTS2UI_RND_MVT_v2f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13240 0 : switch (RetVT.SimpleTy) {
13241 0 : case MVT::i32: return fastEmit_X86ISD_CVTS2UI_RND_MVT_v2f64_MVT_i32_ri(Op0, Op0IsKill, imm1);
13242 0 : case MVT::i64: return fastEmit_X86ISD_CVTS2UI_RND_MVT_v2f64_MVT_i64_ri(Op0, Op0IsKill, imm1);
13243 : default: return 0;
13244 : }
13245 : }
13246 :
13247 0 : unsigned fastEmit_X86ISD_CVTS2UI_RND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13248 0 : switch (VT.SimpleTy) {
13249 0 : case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_RND_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
13250 0 : case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_RND_MVT_v2f64_ri(RetVT, Op0, Op0IsKill, imm1);
13251 : default: return 0;
13252 : }
13253 : }
13254 :
13255 : // FastEmit functions for X86ISD::FSQRT_RND.
13256 :
13257 : unsigned fastEmit_X86ISD_FSQRT_RND_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13258 0 : if (RetVT.SimpleTy != MVT::v16f32)
13259 : return 0;
13260 0 : if ((Subtarget->hasAVX512())) {
13261 0 : return fastEmitInst_ri(X86::VSQRTPSZrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13262 : }
13263 : return 0;
13264 : }
13265 :
13266 : unsigned fastEmit_X86ISD_FSQRT_RND_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13267 0 : if (RetVT.SimpleTy != MVT::v8f64)
13268 : return 0;
13269 0 : if ((Subtarget->hasAVX512())) {
13270 0 : return fastEmitInst_ri(X86::VSQRTPDZrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13271 : }
13272 : return 0;
13273 : }
13274 :
13275 0 : unsigned fastEmit_X86ISD_FSQRT_RND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13276 0 : switch (VT.SimpleTy) {
13277 0 : case MVT::v16f32: return fastEmit_X86ISD_FSQRT_RND_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
13278 0 : case MVT::v8f64: return fastEmit_X86ISD_FSQRT_RND_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
13279 : default: return 0;
13280 : }
13281 : }
13282 :
13283 : // FastEmit functions for X86ISD::KSHIFTL.
13284 :
13285 : unsigned fastEmit_X86ISD_KSHIFTL_MVT_v8i1_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13286 0 : if (RetVT.SimpleTy != MVT::v8i1)
13287 : return 0;
13288 0 : if ((Subtarget->hasDQI())) {
13289 0 : return fastEmitInst_ri(X86::KSHIFTLBri, &X86::VK8RegClass, Op0, Op0IsKill, imm1);
13290 : }
13291 : return 0;
13292 : }
13293 :
13294 : unsigned fastEmit_X86ISD_KSHIFTL_MVT_v16i1_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13295 0 : if (RetVT.SimpleTy != MVT::v16i1)
13296 : return 0;
13297 0 : if ((Subtarget->hasAVX512())) {
13298 0 : return fastEmitInst_ri(X86::KSHIFTLWri, &X86::VK16RegClass, Op0, Op0IsKill, imm1);
13299 : }
13300 : return 0;
13301 : }
13302 :
13303 : unsigned fastEmit_X86ISD_KSHIFTL_MVT_v32i1_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13304 0 : if (RetVT.SimpleTy != MVT::v32i1)
13305 : return 0;
13306 0 : if ((Subtarget->hasBWI())) {
13307 0 : return fastEmitInst_ri(X86::KSHIFTLDri, &X86::VK32RegClass, Op0, Op0IsKill, imm1);
13308 : }
13309 : return 0;
13310 : }
13311 :
13312 : unsigned fastEmit_X86ISD_KSHIFTL_MVT_v64i1_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13313 0 : if (RetVT.SimpleTy != MVT::v64i1)
13314 : return 0;
13315 0 : if ((Subtarget->hasBWI())) {
13316 0 : return fastEmitInst_ri(X86::KSHIFTLQri, &X86::VK64RegClass, Op0, Op0IsKill, imm1);
13317 : }
13318 : return 0;
13319 : }
13320 :
13321 0 : unsigned fastEmit_X86ISD_KSHIFTL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13322 0 : switch (VT.SimpleTy) {
13323 0 : case MVT::v8i1: return fastEmit_X86ISD_KSHIFTL_MVT_v8i1_ri(RetVT, Op0, Op0IsKill, imm1);
13324 0 : case MVT::v16i1: return fastEmit_X86ISD_KSHIFTL_MVT_v16i1_ri(RetVT, Op0, Op0IsKill, imm1);
13325 0 : case MVT::v32i1: return fastEmit_X86ISD_KSHIFTL_MVT_v32i1_ri(RetVT, Op0, Op0IsKill, imm1);
13326 0 : case MVT::v64i1: return fastEmit_X86ISD_KSHIFTL_MVT_v64i1_ri(RetVT, Op0, Op0IsKill, imm1);
13327 : default: return 0;
13328 : }
13329 : }
13330 :
13331 : // FastEmit functions for X86ISD::KSHIFTR.
13332 :
13333 : unsigned fastEmit_X86ISD_KSHIFTR_MVT_v8i1_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13334 0 : if (RetVT.SimpleTy != MVT::v8i1)
13335 : return 0;
13336 0 : if ((Subtarget->hasDQI())) {
13337 0 : return fastEmitInst_ri(X86::KSHIFTRBri, &X86::VK8RegClass, Op0, Op0IsKill, imm1);
13338 : }
13339 : return 0;
13340 : }
13341 :
13342 : unsigned fastEmit_X86ISD_KSHIFTR_MVT_v16i1_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13343 0 : if (RetVT.SimpleTy != MVT::v16i1)
13344 : return 0;
13345 0 : if ((Subtarget->hasAVX512())) {
13346 0 : return fastEmitInst_ri(X86::KSHIFTRWri, &X86::VK16RegClass, Op0, Op0IsKill, imm1);
13347 : }
13348 : return 0;
13349 : }
13350 :
13351 : unsigned fastEmit_X86ISD_KSHIFTR_MVT_v32i1_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13352 0 : if (RetVT.SimpleTy != MVT::v32i1)
13353 : return 0;
13354 0 : if ((Subtarget->hasBWI())) {
13355 0 : return fastEmitInst_ri(X86::KSHIFTRDri, &X86::VK32RegClass, Op0, Op0IsKill, imm1);
13356 : }
13357 : return 0;
13358 : }
13359 :
13360 : unsigned fastEmit_X86ISD_KSHIFTR_MVT_v64i1_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13361 0 : if (RetVT.SimpleTy != MVT::v64i1)
13362 : return 0;
13363 0 : if ((Subtarget->hasBWI())) {
13364 0 : return fastEmitInst_ri(X86::KSHIFTRQri, &X86::VK64RegClass, Op0, Op0IsKill, imm1);
13365 : }
13366 : return 0;
13367 : }
13368 :
13369 0 : unsigned fastEmit_X86ISD_KSHIFTR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13370 0 : switch (VT.SimpleTy) {
13371 0 : case MVT::v8i1: return fastEmit_X86ISD_KSHIFTR_MVT_v8i1_ri(RetVT, Op0, Op0IsKill, imm1);
13372 0 : case MVT::v16i1: return fastEmit_X86ISD_KSHIFTR_MVT_v16i1_ri(RetVT, Op0, Op0IsKill, imm1);
13373 0 : case MVT::v32i1: return fastEmit_X86ISD_KSHIFTR_MVT_v32i1_ri(RetVT, Op0, Op0IsKill, imm1);
13374 0 : case MVT::v64i1: return fastEmit_X86ISD_KSHIFTR_MVT_v64i1_ri(RetVT, Op0, Op0IsKill, imm1);
13375 : default: return 0;
13376 : }
13377 : }
13378 :
13379 : // FastEmit functions for X86ISD::PEXTRB.
13380 :
13381 0 : unsigned fastEmit_X86ISD_PEXTRB_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13382 0 : if (RetVT.SimpleTy != MVT::i32)
13383 : return 0;
13384 0 : if ((Subtarget->hasBWI())) {
13385 0 : return fastEmitInst_ri(X86::VPEXTRBZrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13386 : }
13387 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13388 0 : return fastEmitInst_ri(X86::PEXTRBrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13389 : }
13390 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasBWI())) {
13391 0 : return fastEmitInst_ri(X86::VPEXTRBrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13392 : }
13393 : return 0;
13394 : }
13395 :
13396 : unsigned fastEmit_X86ISD_PEXTRB_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13397 0 : switch (VT.SimpleTy) {
13398 0 : case MVT::v16i8: return fastEmit_X86ISD_PEXTRB_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
13399 : default: return 0;
13400 : }
13401 : }
13402 :
13403 : // FastEmit functions for X86ISD::PEXTRW.
13404 :
13405 0 : unsigned fastEmit_X86ISD_PEXTRW_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13406 0 : if (RetVT.SimpleTy != MVT::i32)
13407 : return 0;
13408 0 : if ((Subtarget->hasBWI())) {
13409 0 : return fastEmitInst_ri(X86::VPEXTRWZrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13410 : }
13411 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13412 0 : return fastEmitInst_ri(X86::PEXTRWrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13413 : }
13414 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasBWI())) {
13415 0 : return fastEmitInst_ri(X86::VPEXTRWrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
13416 : }
13417 : return 0;
13418 : }
13419 :
13420 : unsigned fastEmit_X86ISD_PEXTRW_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13421 0 : switch (VT.SimpleTy) {
13422 0 : case MVT::v8i16: return fastEmit_X86ISD_PEXTRW_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
13423 : default: return 0;
13424 : }
13425 : }
13426 :
13427 : // FastEmit functions for X86ISD::PSHUFD.
13428 :
13429 0 : unsigned fastEmit_X86ISD_PSHUFD_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13430 0 : if (RetVT.SimpleTy != MVT::v4i32)
13431 : return 0;
13432 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13433 0 : return fastEmitInst_ri(X86::VPSHUFDZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13434 : }
13435 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13436 0 : return fastEmitInst_ri(X86::PSHUFDri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13437 : }
13438 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13439 0 : return fastEmitInst_ri(X86::VPSHUFDri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13440 : }
13441 : return 0;
13442 : }
13443 :
13444 0 : unsigned fastEmit_X86ISD_PSHUFD_MVT_v8i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13445 0 : if (RetVT.SimpleTy != MVT::v8i32)
13446 : return 0;
13447 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13448 0 : return fastEmitInst_ri(X86::VPSHUFDZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13449 : }
13450 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13451 0 : return fastEmitInst_ri(X86::VPSHUFDYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
13452 : }
13453 : return 0;
13454 : }
13455 :
13456 : unsigned fastEmit_X86ISD_PSHUFD_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13457 0 : if (RetVT.SimpleTy != MVT::v16i32)
13458 : return 0;
13459 0 : if ((Subtarget->hasAVX512())) {
13460 0 : return fastEmitInst_ri(X86::VPSHUFDZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13461 : }
13462 : return 0;
13463 : }
13464 :
13465 0 : unsigned fastEmit_X86ISD_PSHUFD_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13466 0 : switch (VT.SimpleTy) {
13467 0 : case MVT::v4i32: return fastEmit_X86ISD_PSHUFD_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
13468 0 : case MVT::v8i32: return fastEmit_X86ISD_PSHUFD_MVT_v8i32_ri(RetVT, Op0, Op0IsKill, imm1);
13469 0 : case MVT::v16i32: return fastEmit_X86ISD_PSHUFD_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
13470 : default: return 0;
13471 : }
13472 : }
13473 :
13474 : // FastEmit functions for X86ISD::PSHUFHW.
13475 :
13476 0 : unsigned fastEmit_X86ISD_PSHUFHW_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13477 0 : if (RetVT.SimpleTy != MVT::v8i16)
13478 : return 0;
13479 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13480 0 : return fastEmitInst_ri(X86::VPSHUFHWZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13481 : }
13482 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13483 0 : return fastEmitInst_ri(X86::PSHUFHWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13484 : }
13485 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13486 0 : return fastEmitInst_ri(X86::VPSHUFHWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13487 : }
13488 : return 0;
13489 : }
13490 :
13491 0 : unsigned fastEmit_X86ISD_PSHUFHW_MVT_v16i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13492 0 : if (RetVT.SimpleTy != MVT::v16i16)
13493 : return 0;
13494 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13495 0 : return fastEmitInst_ri(X86::VPSHUFHWZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13496 : }
13497 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13498 0 : return fastEmitInst_ri(X86::VPSHUFHWYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
13499 : }
13500 : return 0;
13501 : }
13502 :
13503 : unsigned fastEmit_X86ISD_PSHUFHW_MVT_v32i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13504 0 : if (RetVT.SimpleTy != MVT::v32i16)
13505 : return 0;
13506 0 : if ((Subtarget->hasBWI())) {
13507 0 : return fastEmitInst_ri(X86::VPSHUFHWZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13508 : }
13509 : return 0;
13510 : }
13511 :
13512 0 : unsigned fastEmit_X86ISD_PSHUFHW_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13513 0 : switch (VT.SimpleTy) {
13514 0 : case MVT::v8i16: return fastEmit_X86ISD_PSHUFHW_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
13515 0 : case MVT::v16i16: return fastEmit_X86ISD_PSHUFHW_MVT_v16i16_ri(RetVT, Op0, Op0IsKill, imm1);
13516 0 : case MVT::v32i16: return fastEmit_X86ISD_PSHUFHW_MVT_v32i16_ri(RetVT, Op0, Op0IsKill, imm1);
13517 : default: return 0;
13518 : }
13519 : }
13520 :
13521 : // FastEmit functions for X86ISD::PSHUFLW.
13522 :
13523 0 : unsigned fastEmit_X86ISD_PSHUFLW_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13524 0 : if (RetVT.SimpleTy != MVT::v8i16)
13525 : return 0;
13526 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13527 0 : return fastEmitInst_ri(X86::VPSHUFLWZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13528 : }
13529 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13530 0 : return fastEmitInst_ri(X86::PSHUFLWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13531 : }
13532 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13533 0 : return fastEmitInst_ri(X86::VPSHUFLWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13534 : }
13535 : return 0;
13536 : }
13537 :
13538 0 : unsigned fastEmit_X86ISD_PSHUFLW_MVT_v16i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13539 0 : if (RetVT.SimpleTy != MVT::v16i16)
13540 : return 0;
13541 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13542 0 : return fastEmitInst_ri(X86::VPSHUFLWZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13543 : }
13544 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13545 0 : return fastEmitInst_ri(X86::VPSHUFLWYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
13546 : }
13547 : return 0;
13548 : }
13549 :
13550 : unsigned fastEmit_X86ISD_PSHUFLW_MVT_v32i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13551 0 : if (RetVT.SimpleTy != MVT::v32i16)
13552 : return 0;
13553 0 : if ((Subtarget->hasBWI())) {
13554 0 : return fastEmitInst_ri(X86::VPSHUFLWZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13555 : }
13556 : return 0;
13557 : }
13558 :
13559 0 : unsigned fastEmit_X86ISD_PSHUFLW_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13560 0 : switch (VT.SimpleTy) {
13561 0 : case MVT::v8i16: return fastEmit_X86ISD_PSHUFLW_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
13562 0 : case MVT::v16i16: return fastEmit_X86ISD_PSHUFLW_MVT_v16i16_ri(RetVT, Op0, Op0IsKill, imm1);
13563 0 : case MVT::v32i16: return fastEmit_X86ISD_PSHUFLW_MVT_v32i16_ri(RetVT, Op0, Op0IsKill, imm1);
13564 : default: return 0;
13565 : }
13566 : }
13567 :
13568 : // FastEmit functions for X86ISD::SINT_TO_FP_RND.
13569 :
13570 : unsigned fastEmit_X86ISD_SINT_TO_FP_RND_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13571 0 : if (RetVT.SimpleTy != MVT::v16f32)
13572 : return 0;
13573 0 : if ((Subtarget->hasAVX512())) {
13574 0 : return fastEmitInst_ri(X86::VCVTDQ2PSZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13575 : }
13576 : return 0;
13577 : }
13578 :
13579 : unsigned fastEmit_X86ISD_SINT_TO_FP_RND_MVT_v8i64_MVT_v8f32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13580 0 : if ((Subtarget->hasDQI())) {
13581 0 : return fastEmitInst_ri(X86::VCVTQQ2PSZrrb, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13582 : }
13583 : return 0;
13584 : }
13585 :
13586 : unsigned fastEmit_X86ISD_SINT_TO_FP_RND_MVT_v8i64_MVT_v8f64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13587 0 : if ((Subtarget->hasDQI())) {
13588 0 : return fastEmitInst_ri(X86::VCVTQQ2PDZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13589 : }
13590 : return 0;
13591 : }
13592 :
13593 0 : unsigned fastEmit_X86ISD_SINT_TO_FP_RND_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13594 0 : switch (RetVT.SimpleTy) {
13595 0 : case MVT::v8f32: return fastEmit_X86ISD_SINT_TO_FP_RND_MVT_v8i64_MVT_v8f32_ri(Op0, Op0IsKill, imm1);
13596 0 : case MVT::v8f64: return fastEmit_X86ISD_SINT_TO_FP_RND_MVT_v8i64_MVT_v8f64_ri(Op0, Op0IsKill, imm1);
13597 : default: return 0;
13598 : }
13599 : }
13600 :
13601 0 : unsigned fastEmit_X86ISD_SINT_TO_FP_RND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13602 0 : switch (VT.SimpleTy) {
13603 0 : case MVT::v16i32: return fastEmit_X86ISD_SINT_TO_FP_RND_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
13604 0 : case MVT::v8i64: return fastEmit_X86ISD_SINT_TO_FP_RND_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
13605 : default: return 0;
13606 : }
13607 : }
13608 :
13609 : // FastEmit functions for X86ISD::UINT_TO_FP_RND.
13610 :
13611 : unsigned fastEmit_X86ISD_UINT_TO_FP_RND_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13612 0 : if (RetVT.SimpleTy != MVT::v16f32)
13613 : return 0;
13614 0 : if ((Subtarget->hasAVX512())) {
13615 0 : return fastEmitInst_ri(X86::VCVTUDQ2PSZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13616 : }
13617 : return 0;
13618 : }
13619 :
13620 : unsigned fastEmit_X86ISD_UINT_TO_FP_RND_MVT_v8i64_MVT_v8f32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13621 0 : if ((Subtarget->hasDQI())) {
13622 0 : return fastEmitInst_ri(X86::VCVTUQQ2PSZrrb, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13623 : }
13624 : return 0;
13625 : }
13626 :
13627 : unsigned fastEmit_X86ISD_UINT_TO_FP_RND_MVT_v8i64_MVT_v8f64_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13628 0 : if ((Subtarget->hasDQI())) {
13629 0 : return fastEmitInst_ri(X86::VCVTUQQ2PDZrrb, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13630 : }
13631 : return 0;
13632 : }
13633 :
13634 0 : unsigned fastEmit_X86ISD_UINT_TO_FP_RND_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13635 0 : switch (RetVT.SimpleTy) {
13636 0 : case MVT::v8f32: return fastEmit_X86ISD_UINT_TO_FP_RND_MVT_v8i64_MVT_v8f32_ri(Op0, Op0IsKill, imm1);
13637 0 : case MVT::v8f64: return fastEmit_X86ISD_UINT_TO_FP_RND_MVT_v8i64_MVT_v8f64_ri(Op0, Op0IsKill, imm1);
13638 : default: return 0;
13639 : }
13640 : }
13641 :
13642 0 : unsigned fastEmit_X86ISD_UINT_TO_FP_RND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13643 0 : switch (VT.SimpleTy) {
13644 0 : case MVT::v16i32: return fastEmit_X86ISD_UINT_TO_FP_RND_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
13645 0 : case MVT::v8i64: return fastEmit_X86ISD_UINT_TO_FP_RND_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
13646 : default: return 0;
13647 : }
13648 : }
13649 :
13650 : // FastEmit functions for X86ISD::VFPCLASS.
13651 :
13652 0 : unsigned fastEmit_X86ISD_VFPCLASS_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13653 0 : if (RetVT.SimpleTy != MVT::v4i1)
13654 : return 0;
13655 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
13656 0 : return fastEmitInst_ri(X86::VFPCLASSPSZ128rr, &X86::VK4RegClass, Op0, Op0IsKill, imm1);
13657 : }
13658 : return 0;
13659 : }
13660 :
13661 0 : unsigned fastEmit_X86ISD_VFPCLASS_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13662 0 : if (RetVT.SimpleTy != MVT::v8i1)
13663 : return 0;
13664 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
13665 0 : return fastEmitInst_ri(X86::VFPCLASSPSZ256rr, &X86::VK8RegClass, Op0, Op0IsKill, imm1);
13666 : }
13667 : return 0;
13668 : }
13669 :
13670 : unsigned fastEmit_X86ISD_VFPCLASS_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13671 0 : if (RetVT.SimpleTy != MVT::v16i1)
13672 : return 0;
13673 0 : if ((Subtarget->hasDQI())) {
13674 0 : return fastEmitInst_ri(X86::VFPCLASSPSZrr, &X86::VK16RegClass, Op0, Op0IsKill, imm1);
13675 : }
13676 : return 0;
13677 : }
13678 :
13679 0 : unsigned fastEmit_X86ISD_VFPCLASS_MVT_v2f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13680 0 : if (RetVT.SimpleTy != MVT::v2i1)
13681 : return 0;
13682 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
13683 0 : return fastEmitInst_ri(X86::VFPCLASSPDZ128rr, &X86::VK2RegClass, Op0, Op0IsKill, imm1);
13684 : }
13685 : return 0;
13686 : }
13687 :
13688 0 : unsigned fastEmit_X86ISD_VFPCLASS_MVT_v4f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13689 0 : if (RetVT.SimpleTy != MVT::v4i1)
13690 : return 0;
13691 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
13692 0 : return fastEmitInst_ri(X86::VFPCLASSPDZ256rr, &X86::VK4RegClass, Op0, Op0IsKill, imm1);
13693 : }
13694 : return 0;
13695 : }
13696 :
13697 : unsigned fastEmit_X86ISD_VFPCLASS_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13698 0 : if (RetVT.SimpleTy != MVT::v8i1)
13699 : return 0;
13700 0 : if ((Subtarget->hasDQI())) {
13701 0 : return fastEmitInst_ri(X86::VFPCLASSPDZrr, &X86::VK8RegClass, Op0, Op0IsKill, imm1);
13702 : }
13703 : return 0;
13704 : }
13705 :
13706 0 : unsigned fastEmit_X86ISD_VFPCLASS_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13707 0 : switch (VT.SimpleTy) {
13708 0 : case MVT::v4f32: return fastEmit_X86ISD_VFPCLASS_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
13709 0 : case MVT::v8f32: return fastEmit_X86ISD_VFPCLASS_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
13710 0 : case MVT::v16f32: return fastEmit_X86ISD_VFPCLASS_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
13711 0 : case MVT::v2f64: return fastEmit_X86ISD_VFPCLASS_MVT_v2f64_ri(RetVT, Op0, Op0IsKill, imm1);
13712 0 : case MVT::v4f64: return fastEmit_X86ISD_VFPCLASS_MVT_v4f64_ri(RetVT, Op0, Op0IsKill, imm1);
13713 0 : case MVT::v8f64: return fastEmit_X86ISD_VFPCLASS_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
13714 : default: return 0;
13715 : }
13716 : }
13717 :
13718 : // FastEmit functions for X86ISD::VFPCLASSS.
13719 :
13720 : unsigned fastEmit_X86ISD_VFPCLASSS_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13721 0 : if (RetVT.SimpleTy != MVT::v1i1)
13722 : return 0;
13723 0 : if ((Subtarget->hasDQI())) {
13724 0 : return fastEmitInst_ri(X86::VFPCLASSSSZrr, &X86::VK1RegClass, Op0, Op0IsKill, imm1);
13725 : }
13726 : return 0;
13727 : }
13728 :
13729 : unsigned fastEmit_X86ISD_VFPCLASSS_MVT_v2f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13730 0 : if (RetVT.SimpleTy != MVT::v1i1)
13731 : return 0;
13732 0 : if ((Subtarget->hasDQI())) {
13733 0 : return fastEmitInst_ri(X86::VFPCLASSSDZrr, &X86::VK1RegClass, Op0, Op0IsKill, imm1);
13734 : }
13735 : return 0;
13736 : }
13737 :
13738 0 : unsigned fastEmit_X86ISD_VFPCLASSS_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13739 0 : switch (VT.SimpleTy) {
13740 0 : case MVT::v4f32: return fastEmit_X86ISD_VFPCLASSS_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
13741 0 : case MVT::v2f64: return fastEmit_X86ISD_VFPCLASSS_MVT_v2f64_ri(RetVT, Op0, Op0IsKill, imm1);
13742 : default: return 0;
13743 : }
13744 : }
13745 :
13746 : // FastEmit functions for X86ISD::VFPROUND_RND.
13747 :
13748 : unsigned fastEmit_X86ISD_VFPROUND_RND_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13749 0 : if (RetVT.SimpleTy != MVT::v8f32)
13750 : return 0;
13751 0 : if ((Subtarget->hasAVX512())) {
13752 0 : return fastEmitInst_ri(X86::VCVTPD2PSZrrb, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13753 : }
13754 : return 0;
13755 : }
13756 :
13757 0 : unsigned fastEmit_X86ISD_VFPROUND_RND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13758 0 : switch (VT.SimpleTy) {
13759 0 : case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_RND_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
13760 : default: return 0;
13761 : }
13762 : }
13763 :
13764 : // FastEmit functions for X86ISD::VGETMANT.
13765 :
13766 0 : unsigned fastEmit_X86ISD_VGETMANT_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13767 0 : if (RetVT.SimpleTy != MVT::v4f32)
13768 : return 0;
13769 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13770 0 : return fastEmitInst_ri(X86::VGETMANTPSZ128rri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13771 : }
13772 : return 0;
13773 : }
13774 :
13775 0 : unsigned fastEmit_X86ISD_VGETMANT_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13776 0 : if (RetVT.SimpleTy != MVT::v8f32)
13777 : return 0;
13778 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13779 0 : return fastEmitInst_ri(X86::VGETMANTPSZ256rri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13780 : }
13781 : return 0;
13782 : }
13783 :
13784 : unsigned fastEmit_X86ISD_VGETMANT_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13785 0 : if (RetVT.SimpleTy != MVT::v16f32)
13786 : return 0;
13787 0 : if ((Subtarget->hasAVX512())) {
13788 0 : return fastEmitInst_ri(X86::VGETMANTPSZrri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13789 : }
13790 : return 0;
13791 : }
13792 :
13793 0 : unsigned fastEmit_X86ISD_VGETMANT_MVT_v2f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13794 0 : if (RetVT.SimpleTy != MVT::v2f64)
13795 : return 0;
13796 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13797 0 : return fastEmitInst_ri(X86::VGETMANTPDZ128rri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13798 : }
13799 : return 0;
13800 : }
13801 :
13802 0 : unsigned fastEmit_X86ISD_VGETMANT_MVT_v4f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13803 0 : if (RetVT.SimpleTy != MVT::v4f64)
13804 : return 0;
13805 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13806 0 : return fastEmitInst_ri(X86::VGETMANTPDZ256rri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13807 : }
13808 : return 0;
13809 : }
13810 :
13811 : unsigned fastEmit_X86ISD_VGETMANT_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13812 0 : if (RetVT.SimpleTy != MVT::v8f64)
13813 : return 0;
13814 0 : if ((Subtarget->hasAVX512())) {
13815 0 : return fastEmitInst_ri(X86::VGETMANTPDZrri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13816 : }
13817 : return 0;
13818 : }
13819 :
13820 0 : unsigned fastEmit_X86ISD_VGETMANT_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13821 0 : switch (VT.SimpleTy) {
13822 0 : case MVT::v4f32: return fastEmit_X86ISD_VGETMANT_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
13823 0 : case MVT::v8f32: return fastEmit_X86ISD_VGETMANT_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
13824 0 : case MVT::v16f32: return fastEmit_X86ISD_VGETMANT_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
13825 0 : case MVT::v2f64: return fastEmit_X86ISD_VGETMANT_MVT_v2f64_ri(RetVT, Op0, Op0IsKill, imm1);
13826 0 : case MVT::v4f64: return fastEmit_X86ISD_VGETMANT_MVT_v4f64_ri(RetVT, Op0, Op0IsKill, imm1);
13827 0 : case MVT::v8f64: return fastEmit_X86ISD_VGETMANT_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
13828 : default: return 0;
13829 : }
13830 : }
13831 :
13832 : // FastEmit functions for X86ISD::VPERMI.
13833 :
13834 0 : unsigned fastEmit_X86ISD_VPERMI_MVT_v4i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13835 0 : if (RetVT.SimpleTy != MVT::v4i64)
13836 : return 0;
13837 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13838 0 : return fastEmitInst_ri(X86::VPERMQZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13839 : }
13840 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13841 0 : return fastEmitInst_ri(X86::VPERMQYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
13842 : }
13843 : return 0;
13844 : }
13845 :
13846 : unsigned fastEmit_X86ISD_VPERMI_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13847 0 : if (RetVT.SimpleTy != MVT::v8i64)
13848 : return 0;
13849 0 : if ((Subtarget->hasAVX512())) {
13850 0 : return fastEmitInst_ri(X86::VPERMQZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13851 : }
13852 : return 0;
13853 : }
13854 :
13855 0 : unsigned fastEmit_X86ISD_VPERMI_MVT_v4f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13856 0 : if (RetVT.SimpleTy != MVT::v4f64)
13857 : return 0;
13858 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13859 0 : return fastEmitInst_ri(X86::VPERMPDZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13860 : }
13861 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13862 0 : return fastEmitInst_ri(X86::VPERMPDYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
13863 : }
13864 : return 0;
13865 : }
13866 :
13867 : unsigned fastEmit_X86ISD_VPERMI_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13868 0 : if (RetVT.SimpleTy != MVT::v8f64)
13869 : return 0;
13870 0 : if ((Subtarget->hasAVX512())) {
13871 0 : return fastEmitInst_ri(X86::VPERMPDZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13872 : }
13873 : return 0;
13874 : }
13875 :
13876 0 : unsigned fastEmit_X86ISD_VPERMI_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13877 0 : switch (VT.SimpleTy) {
13878 0 : case MVT::v4i64: return fastEmit_X86ISD_VPERMI_MVT_v4i64_ri(RetVT, Op0, Op0IsKill, imm1);
13879 0 : case MVT::v8i64: return fastEmit_X86ISD_VPERMI_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
13880 0 : case MVT::v4f64: return fastEmit_X86ISD_VPERMI_MVT_v4f64_ri(RetVT, Op0, Op0IsKill, imm1);
13881 0 : case MVT::v8f64: return fastEmit_X86ISD_VPERMI_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
13882 : default: return 0;
13883 : }
13884 : }
13885 :
13886 : // FastEmit functions for X86ISD::VPERMILPI.
13887 :
13888 0 : unsigned fastEmit_X86ISD_VPERMILPI_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13889 0 : if (RetVT.SimpleTy != MVT::v4f32)
13890 : return 0;
13891 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13892 0 : return fastEmitInst_ri(X86::VPERMILPSZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13893 : }
13894 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13895 0 : return fastEmitInst_ri(X86::VPERMILPSri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13896 : }
13897 : return 0;
13898 : }
13899 :
13900 0 : unsigned fastEmit_X86ISD_VPERMILPI_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13901 0 : if (RetVT.SimpleTy != MVT::v8f32)
13902 : return 0;
13903 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13904 0 : return fastEmitInst_ri(X86::VPERMILPSZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13905 : }
13906 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13907 0 : return fastEmitInst_ri(X86::VPERMILPSYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
13908 : }
13909 : return 0;
13910 : }
13911 :
13912 : unsigned fastEmit_X86ISD_VPERMILPI_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13913 0 : if (RetVT.SimpleTy != MVT::v16f32)
13914 : return 0;
13915 0 : if ((Subtarget->hasAVX512())) {
13916 0 : return fastEmitInst_ri(X86::VPERMILPSZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13917 : }
13918 : return 0;
13919 : }
13920 :
13921 0 : unsigned fastEmit_X86ISD_VPERMILPI_MVT_v2f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13922 0 : if (RetVT.SimpleTy != MVT::v2f64)
13923 : return 0;
13924 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13925 0 : return fastEmitInst_ri(X86::VPERMILPDZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13926 : }
13927 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13928 0 : return fastEmitInst_ri(X86::VPERMILPDri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
13929 : }
13930 : return 0;
13931 : }
13932 :
13933 0 : unsigned fastEmit_X86ISD_VPERMILPI_MVT_v4f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13934 0 : if (RetVT.SimpleTy != MVT::v4f64)
13935 : return 0;
13936 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13937 0 : return fastEmitInst_ri(X86::VPERMILPDZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13938 : }
13939 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13940 0 : return fastEmitInst_ri(X86::VPERMILPDYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
13941 : }
13942 : return 0;
13943 : }
13944 :
13945 : unsigned fastEmit_X86ISD_VPERMILPI_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13946 0 : if (RetVT.SimpleTy != MVT::v8f64)
13947 : return 0;
13948 0 : if ((Subtarget->hasAVX512())) {
13949 0 : return fastEmitInst_ri(X86::VPERMILPDZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13950 : }
13951 : return 0;
13952 : }
13953 :
13954 0 : unsigned fastEmit_X86ISD_VPERMILPI_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13955 0 : switch (VT.SimpleTy) {
13956 0 : case MVT::v4f32: return fastEmit_X86ISD_VPERMILPI_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
13957 0 : case MVT::v8f32: return fastEmit_X86ISD_VPERMILPI_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
13958 0 : case MVT::v16f32: return fastEmit_X86ISD_VPERMILPI_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
13959 0 : case MVT::v2f64: return fastEmit_X86ISD_VPERMILPI_MVT_v2f64_ri(RetVT, Op0, Op0IsKill, imm1);
13960 0 : case MVT::v4f64: return fastEmit_X86ISD_VPERMILPI_MVT_v4f64_ri(RetVT, Op0, Op0IsKill, imm1);
13961 0 : case MVT::v8f64: return fastEmit_X86ISD_VPERMILPI_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
13962 : default: return 0;
13963 : }
13964 : }
13965 :
13966 : // FastEmit functions for X86ISD::VREDUCE.
13967 :
13968 0 : unsigned fastEmit_X86ISD_VREDUCE_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13969 0 : if (RetVT.SimpleTy != MVT::v4f32)
13970 : return 0;
13971 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
13972 0 : return fastEmitInst_ri(X86::VREDUCEPSZ128rri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
13973 : }
13974 : return 0;
13975 : }
13976 :
13977 0 : unsigned fastEmit_X86ISD_VREDUCE_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13978 0 : if (RetVT.SimpleTy != MVT::v8f32)
13979 : return 0;
13980 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
13981 0 : return fastEmitInst_ri(X86::VREDUCEPSZ256rri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
13982 : }
13983 : return 0;
13984 : }
13985 :
13986 : unsigned fastEmit_X86ISD_VREDUCE_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13987 0 : if (RetVT.SimpleTy != MVT::v16f32)
13988 : return 0;
13989 0 : if ((Subtarget->hasDQI())) {
13990 0 : return fastEmitInst_ri(X86::VREDUCEPSZrri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
13991 : }
13992 : return 0;
13993 : }
13994 :
13995 0 : unsigned fastEmit_X86ISD_VREDUCE_MVT_v2f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
13996 0 : if (RetVT.SimpleTy != MVT::v2f64)
13997 : return 0;
13998 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
13999 0 : return fastEmitInst_ri(X86::VREDUCEPDZ128rri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14000 : }
14001 : return 0;
14002 : }
14003 :
14004 0 : unsigned fastEmit_X86ISD_VREDUCE_MVT_v4f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14005 0 : if (RetVT.SimpleTy != MVT::v4f64)
14006 : return 0;
14007 0 : if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
14008 0 : return fastEmitInst_ri(X86::VREDUCEPDZ256rri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14009 : }
14010 : return 0;
14011 : }
14012 :
14013 : unsigned fastEmit_X86ISD_VREDUCE_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14014 0 : if (RetVT.SimpleTy != MVT::v8f64)
14015 : return 0;
14016 0 : if ((Subtarget->hasDQI())) {
14017 0 : return fastEmitInst_ri(X86::VREDUCEPDZrri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14018 : }
14019 : return 0;
14020 : }
14021 :
14022 0 : unsigned fastEmit_X86ISD_VREDUCE_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14023 0 : switch (VT.SimpleTy) {
14024 0 : case MVT::v4f32: return fastEmit_X86ISD_VREDUCE_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
14025 0 : case MVT::v8f32: return fastEmit_X86ISD_VREDUCE_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
14026 0 : case MVT::v16f32: return fastEmit_X86ISD_VREDUCE_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
14027 0 : case MVT::v2f64: return fastEmit_X86ISD_VREDUCE_MVT_v2f64_ri(RetVT, Op0, Op0IsKill, imm1);
14028 0 : case MVT::v4f64: return fastEmit_X86ISD_VREDUCE_MVT_v4f64_ri(RetVT, Op0, Op0IsKill, imm1);
14029 0 : case MVT::v8f64: return fastEmit_X86ISD_VREDUCE_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
14030 : default: return 0;
14031 : }
14032 : }
14033 :
14034 : // FastEmit functions for X86ISD::VRNDSCALE.
14035 :
14036 0 : unsigned fastEmit_X86ISD_VRNDSCALE_MVT_v4f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14037 0 : if (RetVT.SimpleTy != MVT::v4f32)
14038 : return 0;
14039 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14040 0 : return fastEmitInst_ri(X86::VRNDSCALEPSZ128rri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14041 : }
14042 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14043 0 : return fastEmitInst_ri(X86::ROUNDPSr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14044 : }
14045 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14046 0 : return fastEmitInst_ri(X86::VROUNDPSr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14047 : }
14048 : return 0;
14049 : }
14050 :
14051 0 : unsigned fastEmit_X86ISD_VRNDSCALE_MVT_v8f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14052 0 : if (RetVT.SimpleTy != MVT::v8f32)
14053 : return 0;
14054 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14055 0 : return fastEmitInst_ri(X86::VRNDSCALEPSZ256rri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14056 : }
14057 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14058 0 : return fastEmitInst_ri(X86::VROUNDPSYr, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14059 : }
14060 : return 0;
14061 : }
14062 :
14063 : unsigned fastEmit_X86ISD_VRNDSCALE_MVT_v16f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14064 0 : if (RetVT.SimpleTy != MVT::v16f32)
14065 : return 0;
14066 0 : if ((Subtarget->hasAVX512())) {
14067 0 : return fastEmitInst_ri(X86::VRNDSCALEPSZrri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14068 : }
14069 : return 0;
14070 : }
14071 :
14072 0 : unsigned fastEmit_X86ISD_VRNDSCALE_MVT_v2f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14073 0 : if (RetVT.SimpleTy != MVT::v2f64)
14074 : return 0;
14075 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14076 0 : return fastEmitInst_ri(X86::VRNDSCALEPDZ128rri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14077 : }
14078 0 : if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
14079 0 : return fastEmitInst_ri(X86::ROUNDPDr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14080 : }
14081 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14082 0 : return fastEmitInst_ri(X86::VROUNDPDr, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14083 : }
14084 : return 0;
14085 : }
14086 :
14087 0 : unsigned fastEmit_X86ISD_VRNDSCALE_MVT_v4f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14088 0 : if (RetVT.SimpleTy != MVT::v4f64)
14089 : return 0;
14090 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14091 0 : return fastEmitInst_ri(X86::VRNDSCALEPDZ256rri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14092 : }
14093 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14094 0 : return fastEmitInst_ri(X86::VROUNDPDYr, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14095 : }
14096 : return 0;
14097 : }
14098 :
14099 : unsigned fastEmit_X86ISD_VRNDSCALE_MVT_v8f64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14100 0 : if (RetVT.SimpleTy != MVT::v8f64)
14101 : return 0;
14102 0 : if ((Subtarget->hasAVX512())) {
14103 0 : return fastEmitInst_ri(X86::VRNDSCALEPDZrri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14104 : }
14105 : return 0;
14106 : }
14107 :
14108 0 : unsigned fastEmit_X86ISD_VRNDSCALE_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14109 0 : switch (VT.SimpleTy) {
14110 0 : case MVT::v4f32: return fastEmit_X86ISD_VRNDSCALE_MVT_v4f32_ri(RetVT, Op0, Op0IsKill, imm1);
14111 0 : case MVT::v8f32: return fastEmit_X86ISD_VRNDSCALE_MVT_v8f32_ri(RetVT, Op0, Op0IsKill, imm1);
14112 0 : case MVT::v16f32: return fastEmit_X86ISD_VRNDSCALE_MVT_v16f32_ri(RetVT, Op0, Op0IsKill, imm1);
14113 0 : case MVT::v2f64: return fastEmit_X86ISD_VRNDSCALE_MVT_v2f64_ri(RetVT, Op0, Op0IsKill, imm1);
14114 0 : case MVT::v4f64: return fastEmit_X86ISD_VRNDSCALE_MVT_v4f64_ri(RetVT, Op0, Op0IsKill, imm1);
14115 0 : case MVT::v8f64: return fastEmit_X86ISD_VRNDSCALE_MVT_v8f64_ri(RetVT, Op0, Op0IsKill, imm1);
14116 : default: return 0;
14117 : }
14118 : }
14119 :
14120 : // FastEmit functions for X86ISD::VROTLI.
14121 :
14122 : unsigned fastEmit_X86ISD_VROTLI_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14123 0 : if (RetVT.SimpleTy != MVT::v16i8)
14124 : return 0;
14125 0 : if ((Subtarget->hasXOP())) {
14126 0 : return fastEmitInst_ri(X86::VPROTBri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14127 : }
14128 : return 0;
14129 : }
14130 :
14131 : unsigned fastEmit_X86ISD_VROTLI_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14132 0 : if (RetVT.SimpleTy != MVT::v8i16)
14133 : return 0;
14134 0 : if ((Subtarget->hasXOP())) {
14135 0 : return fastEmitInst_ri(X86::VPROTWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14136 : }
14137 : return 0;
14138 : }
14139 :
14140 0 : unsigned fastEmit_X86ISD_VROTLI_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14141 0 : if (RetVT.SimpleTy != MVT::v4i32)
14142 : return 0;
14143 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14144 0 : return fastEmitInst_ri(X86::VPROLDZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14145 : }
14146 0 : if ((Subtarget->hasXOP())) {
14147 0 : return fastEmitInst_ri(X86::VPROTDri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14148 : }
14149 : return 0;
14150 : }
14151 :
14152 0 : unsigned fastEmit_X86ISD_VROTLI_MVT_v8i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14153 0 : if (RetVT.SimpleTy != MVT::v8i32)
14154 : return 0;
14155 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14156 0 : return fastEmitInst_ri(X86::VPROLDZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14157 : }
14158 : return 0;
14159 : }
14160 :
14161 : unsigned fastEmit_X86ISD_VROTLI_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14162 0 : if (RetVT.SimpleTy != MVT::v16i32)
14163 : return 0;
14164 0 : if ((Subtarget->hasAVX512())) {
14165 0 : return fastEmitInst_ri(X86::VPROLDZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14166 : }
14167 : return 0;
14168 : }
14169 :
14170 0 : unsigned fastEmit_X86ISD_VROTLI_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14171 0 : if (RetVT.SimpleTy != MVT::v2i64)
14172 : return 0;
14173 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14174 0 : return fastEmitInst_ri(X86::VPROLQZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14175 : }
14176 0 : if ((Subtarget->hasXOP())) {
14177 0 : return fastEmitInst_ri(X86::VPROTQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14178 : }
14179 : return 0;
14180 : }
14181 :
14182 0 : unsigned fastEmit_X86ISD_VROTLI_MVT_v4i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14183 0 : if (RetVT.SimpleTy != MVT::v4i64)
14184 : return 0;
14185 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14186 0 : return fastEmitInst_ri(X86::VPROLQZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14187 : }
14188 : return 0;
14189 : }
14190 :
14191 : unsigned fastEmit_X86ISD_VROTLI_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14192 0 : if (RetVT.SimpleTy != MVT::v8i64)
14193 : return 0;
14194 0 : if ((Subtarget->hasAVX512())) {
14195 0 : return fastEmitInst_ri(X86::VPROLQZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14196 : }
14197 : return 0;
14198 : }
14199 :
14200 0 : unsigned fastEmit_X86ISD_VROTLI_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14201 0 : switch (VT.SimpleTy) {
14202 0 : case MVT::v16i8: return fastEmit_X86ISD_VROTLI_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
14203 0 : case MVT::v8i16: return fastEmit_X86ISD_VROTLI_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
14204 0 : case MVT::v4i32: return fastEmit_X86ISD_VROTLI_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
14205 0 : case MVT::v8i32: return fastEmit_X86ISD_VROTLI_MVT_v8i32_ri(RetVT, Op0, Op0IsKill, imm1);
14206 0 : case MVT::v16i32: return fastEmit_X86ISD_VROTLI_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
14207 0 : case MVT::v2i64: return fastEmit_X86ISD_VROTLI_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
14208 0 : case MVT::v4i64: return fastEmit_X86ISD_VROTLI_MVT_v4i64_ri(RetVT, Op0, Op0IsKill, imm1);
14209 0 : case MVT::v8i64: return fastEmit_X86ISD_VROTLI_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
14210 : default: return 0;
14211 : }
14212 : }
14213 :
14214 : // FastEmit functions for X86ISD::VROTRI.
14215 :
14216 0 : unsigned fastEmit_X86ISD_VROTRI_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14217 0 : if (RetVT.SimpleTy != MVT::v4i32)
14218 : return 0;
14219 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14220 0 : return fastEmitInst_ri(X86::VPRORDZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14221 : }
14222 : return 0;
14223 : }
14224 :
14225 0 : unsigned fastEmit_X86ISD_VROTRI_MVT_v8i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14226 0 : if (RetVT.SimpleTy != MVT::v8i32)
14227 : return 0;
14228 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14229 0 : return fastEmitInst_ri(X86::VPRORDZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14230 : }
14231 : return 0;
14232 : }
14233 :
14234 : unsigned fastEmit_X86ISD_VROTRI_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14235 0 : if (RetVT.SimpleTy != MVT::v16i32)
14236 : return 0;
14237 0 : if ((Subtarget->hasAVX512())) {
14238 0 : return fastEmitInst_ri(X86::VPRORDZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14239 : }
14240 : return 0;
14241 : }
14242 :
14243 0 : unsigned fastEmit_X86ISD_VROTRI_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14244 0 : if (RetVT.SimpleTy != MVT::v2i64)
14245 : return 0;
14246 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14247 0 : return fastEmitInst_ri(X86::VPRORQZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14248 : }
14249 : return 0;
14250 : }
14251 :
14252 0 : unsigned fastEmit_X86ISD_VROTRI_MVT_v4i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14253 0 : if (RetVT.SimpleTy != MVT::v4i64)
14254 : return 0;
14255 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14256 0 : return fastEmitInst_ri(X86::VPRORQZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14257 : }
14258 : return 0;
14259 : }
14260 :
14261 : unsigned fastEmit_X86ISD_VROTRI_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14262 0 : if (RetVT.SimpleTy != MVT::v8i64)
14263 : return 0;
14264 0 : if ((Subtarget->hasAVX512())) {
14265 0 : return fastEmitInst_ri(X86::VPRORQZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14266 : }
14267 : return 0;
14268 : }
14269 :
14270 0 : unsigned fastEmit_X86ISD_VROTRI_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14271 0 : switch (VT.SimpleTy) {
14272 0 : case MVT::v4i32: return fastEmit_X86ISD_VROTRI_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
14273 0 : case MVT::v8i32: return fastEmit_X86ISD_VROTRI_MVT_v8i32_ri(RetVT, Op0, Op0IsKill, imm1);
14274 0 : case MVT::v16i32: return fastEmit_X86ISD_VROTRI_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
14275 0 : case MVT::v2i64: return fastEmit_X86ISD_VROTRI_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
14276 0 : case MVT::v4i64: return fastEmit_X86ISD_VROTRI_MVT_v4i64_ri(RetVT, Op0, Op0IsKill, imm1);
14277 0 : case MVT::v8i64: return fastEmit_X86ISD_VROTRI_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
14278 : default: return 0;
14279 : }
14280 : }
14281 :
14282 : // FastEmit functions for X86ISD::VSHLDQ.
14283 :
14284 0 : unsigned fastEmit_X86ISD_VSHLDQ_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14285 0 : if (RetVT.SimpleTy != MVT::v16i8)
14286 : return 0;
14287 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14288 0 : return fastEmitInst_ri(X86::VPSLLDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14289 : }
14290 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14291 0 : return fastEmitInst_ri(X86::PSLLDQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14292 : }
14293 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14294 0 : return fastEmitInst_ri(X86::VPSLLDQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14295 : }
14296 : return 0;
14297 : }
14298 :
14299 0 : unsigned fastEmit_X86ISD_VSHLDQ_MVT_v32i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14300 0 : if (RetVT.SimpleTy != MVT::v32i8)
14301 : return 0;
14302 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14303 0 : return fastEmitInst_ri(X86::VPSLLDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14304 : }
14305 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14306 0 : return fastEmitInst_ri(X86::VPSLLDQYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14307 : }
14308 : return 0;
14309 : }
14310 :
14311 : unsigned fastEmit_X86ISD_VSHLDQ_MVT_v64i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14312 0 : if (RetVT.SimpleTy != MVT::v64i8)
14313 : return 0;
14314 0 : if ((Subtarget->hasBWI())) {
14315 0 : return fastEmitInst_ri(X86::VPSLLDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14316 : }
14317 : return 0;
14318 : }
14319 :
14320 0 : unsigned fastEmit_X86ISD_VSHLDQ_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14321 0 : switch (VT.SimpleTy) {
14322 0 : case MVT::v16i8: return fastEmit_X86ISD_VSHLDQ_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
14323 0 : case MVT::v32i8: return fastEmit_X86ISD_VSHLDQ_MVT_v32i8_ri(RetVT, Op0, Op0IsKill, imm1);
14324 0 : case MVT::v64i8: return fastEmit_X86ISD_VSHLDQ_MVT_v64i8_ri(RetVT, Op0, Op0IsKill, imm1);
14325 : default: return 0;
14326 : }
14327 : }
14328 :
14329 : // FastEmit functions for X86ISD::VSHLI.
14330 :
14331 0 : unsigned fastEmit_X86ISD_VSHLI_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14332 0 : if (RetVT.SimpleTy != MVT::v8i16)
14333 : return 0;
14334 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14335 0 : return fastEmitInst_ri(X86::VPSLLWZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14336 : }
14337 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14338 0 : return fastEmitInst_ri(X86::PSLLWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14339 : }
14340 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14341 0 : return fastEmitInst_ri(X86::VPSLLWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14342 : }
14343 : return 0;
14344 : }
14345 :
14346 0 : unsigned fastEmit_X86ISD_VSHLI_MVT_v16i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14347 0 : if (RetVT.SimpleTy != MVT::v16i16)
14348 : return 0;
14349 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14350 0 : return fastEmitInst_ri(X86::VPSLLWZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14351 : }
14352 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14353 0 : return fastEmitInst_ri(X86::VPSLLWYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14354 : }
14355 : return 0;
14356 : }
14357 :
14358 : unsigned fastEmit_X86ISD_VSHLI_MVT_v32i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14359 0 : if (RetVT.SimpleTy != MVT::v32i16)
14360 : return 0;
14361 0 : if ((Subtarget->hasBWI())) {
14362 0 : return fastEmitInst_ri(X86::VPSLLWZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14363 : }
14364 : return 0;
14365 : }
14366 :
14367 0 : unsigned fastEmit_X86ISD_VSHLI_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14368 0 : if (RetVT.SimpleTy != MVT::v4i32)
14369 : return 0;
14370 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14371 0 : return fastEmitInst_ri(X86::VPSLLDZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14372 : }
14373 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14374 0 : return fastEmitInst_ri(X86::PSLLDri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14375 : }
14376 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14377 0 : return fastEmitInst_ri(X86::VPSLLDri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14378 : }
14379 : return 0;
14380 : }
14381 :
14382 0 : unsigned fastEmit_X86ISD_VSHLI_MVT_v8i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14383 0 : if (RetVT.SimpleTy != MVT::v8i32)
14384 : return 0;
14385 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14386 0 : return fastEmitInst_ri(X86::VPSLLDZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14387 : }
14388 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14389 0 : return fastEmitInst_ri(X86::VPSLLDYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14390 : }
14391 : return 0;
14392 : }
14393 :
14394 : unsigned fastEmit_X86ISD_VSHLI_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14395 0 : if (RetVT.SimpleTy != MVT::v16i32)
14396 : return 0;
14397 0 : if ((Subtarget->hasAVX512())) {
14398 0 : return fastEmitInst_ri(X86::VPSLLDZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14399 : }
14400 : return 0;
14401 : }
14402 :
14403 0 : unsigned fastEmit_X86ISD_VSHLI_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14404 0 : if (RetVT.SimpleTy != MVT::v2i64)
14405 : return 0;
14406 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14407 0 : return fastEmitInst_ri(X86::VPSLLQZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14408 : }
14409 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14410 0 : return fastEmitInst_ri(X86::PSLLQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14411 : }
14412 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14413 0 : return fastEmitInst_ri(X86::VPSLLQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14414 : }
14415 : return 0;
14416 : }
14417 :
14418 0 : unsigned fastEmit_X86ISD_VSHLI_MVT_v4i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14419 0 : if (RetVT.SimpleTy != MVT::v4i64)
14420 : return 0;
14421 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14422 0 : return fastEmitInst_ri(X86::VPSLLQZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14423 : }
14424 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14425 0 : return fastEmitInst_ri(X86::VPSLLQYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14426 : }
14427 : return 0;
14428 : }
14429 :
14430 : unsigned fastEmit_X86ISD_VSHLI_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14431 0 : if (RetVT.SimpleTy != MVT::v8i64)
14432 : return 0;
14433 0 : if ((Subtarget->hasAVX512())) {
14434 0 : return fastEmitInst_ri(X86::VPSLLQZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14435 : }
14436 : return 0;
14437 : }
14438 :
14439 0 : unsigned fastEmit_X86ISD_VSHLI_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14440 0 : switch (VT.SimpleTy) {
14441 0 : case MVT::v8i16: return fastEmit_X86ISD_VSHLI_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
14442 0 : case MVT::v16i16: return fastEmit_X86ISD_VSHLI_MVT_v16i16_ri(RetVT, Op0, Op0IsKill, imm1);
14443 0 : case MVT::v32i16: return fastEmit_X86ISD_VSHLI_MVT_v32i16_ri(RetVT, Op0, Op0IsKill, imm1);
14444 0 : case MVT::v4i32: return fastEmit_X86ISD_VSHLI_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
14445 0 : case MVT::v8i32: return fastEmit_X86ISD_VSHLI_MVT_v8i32_ri(RetVT, Op0, Op0IsKill, imm1);
14446 0 : case MVT::v16i32: return fastEmit_X86ISD_VSHLI_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
14447 0 : case MVT::v2i64: return fastEmit_X86ISD_VSHLI_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
14448 0 : case MVT::v4i64: return fastEmit_X86ISD_VSHLI_MVT_v4i64_ri(RetVT, Op0, Op0IsKill, imm1);
14449 0 : case MVT::v8i64: return fastEmit_X86ISD_VSHLI_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
14450 : default: return 0;
14451 : }
14452 : }
14453 :
14454 : // FastEmit functions for X86ISD::VSRAI.
14455 :
14456 0 : unsigned fastEmit_X86ISD_VSRAI_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14457 0 : if (RetVT.SimpleTy != MVT::v8i16)
14458 : return 0;
14459 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14460 0 : return fastEmitInst_ri(X86::VPSRAWZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14461 : }
14462 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14463 0 : return fastEmitInst_ri(X86::PSRAWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14464 : }
14465 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14466 0 : return fastEmitInst_ri(X86::VPSRAWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14467 : }
14468 : return 0;
14469 : }
14470 :
14471 0 : unsigned fastEmit_X86ISD_VSRAI_MVT_v16i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14472 0 : if (RetVT.SimpleTy != MVT::v16i16)
14473 : return 0;
14474 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14475 0 : return fastEmitInst_ri(X86::VPSRAWZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14476 : }
14477 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14478 0 : return fastEmitInst_ri(X86::VPSRAWYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14479 : }
14480 : return 0;
14481 : }
14482 :
14483 : unsigned fastEmit_X86ISD_VSRAI_MVT_v32i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14484 0 : if (RetVT.SimpleTy != MVT::v32i16)
14485 : return 0;
14486 0 : if ((Subtarget->hasBWI())) {
14487 0 : return fastEmitInst_ri(X86::VPSRAWZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14488 : }
14489 : return 0;
14490 : }
14491 :
14492 0 : unsigned fastEmit_X86ISD_VSRAI_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14493 0 : if (RetVT.SimpleTy != MVT::v4i32)
14494 : return 0;
14495 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14496 0 : return fastEmitInst_ri(X86::VPSRADZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14497 : }
14498 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14499 0 : return fastEmitInst_ri(X86::PSRADri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14500 : }
14501 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14502 0 : return fastEmitInst_ri(X86::VPSRADri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14503 : }
14504 : return 0;
14505 : }
14506 :
14507 0 : unsigned fastEmit_X86ISD_VSRAI_MVT_v8i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14508 0 : if (RetVT.SimpleTy != MVT::v8i32)
14509 : return 0;
14510 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14511 0 : return fastEmitInst_ri(X86::VPSRADZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14512 : }
14513 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14514 0 : return fastEmitInst_ri(X86::VPSRADYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14515 : }
14516 : return 0;
14517 : }
14518 :
14519 : unsigned fastEmit_X86ISD_VSRAI_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14520 0 : if (RetVT.SimpleTy != MVT::v16i32)
14521 : return 0;
14522 0 : if ((Subtarget->hasAVX512())) {
14523 0 : return fastEmitInst_ri(X86::VPSRADZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14524 : }
14525 : return 0;
14526 : }
14527 :
14528 0 : unsigned fastEmit_X86ISD_VSRAI_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14529 0 : if (RetVT.SimpleTy != MVT::v2i64)
14530 : return 0;
14531 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14532 0 : return fastEmitInst_ri(X86::VPSRAQZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14533 : }
14534 : return 0;
14535 : }
14536 :
14537 0 : unsigned fastEmit_X86ISD_VSRAI_MVT_v4i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14538 0 : if (RetVT.SimpleTy != MVT::v4i64)
14539 : return 0;
14540 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14541 0 : return fastEmitInst_ri(X86::VPSRAQZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14542 : }
14543 : return 0;
14544 : }
14545 :
14546 : unsigned fastEmit_X86ISD_VSRAI_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14547 0 : if (RetVT.SimpleTy != MVT::v8i64)
14548 : return 0;
14549 0 : if ((Subtarget->hasAVX512())) {
14550 0 : return fastEmitInst_ri(X86::VPSRAQZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14551 : }
14552 : return 0;
14553 : }
14554 :
14555 0 : unsigned fastEmit_X86ISD_VSRAI_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14556 0 : switch (VT.SimpleTy) {
14557 0 : case MVT::v8i16: return fastEmit_X86ISD_VSRAI_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
14558 0 : case MVT::v16i16: return fastEmit_X86ISD_VSRAI_MVT_v16i16_ri(RetVT, Op0, Op0IsKill, imm1);
14559 0 : case MVT::v32i16: return fastEmit_X86ISD_VSRAI_MVT_v32i16_ri(RetVT, Op0, Op0IsKill, imm1);
14560 0 : case MVT::v4i32: return fastEmit_X86ISD_VSRAI_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
14561 0 : case MVT::v8i32: return fastEmit_X86ISD_VSRAI_MVT_v8i32_ri(RetVT, Op0, Op0IsKill, imm1);
14562 0 : case MVT::v16i32: return fastEmit_X86ISD_VSRAI_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
14563 0 : case MVT::v2i64: return fastEmit_X86ISD_VSRAI_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
14564 0 : case MVT::v4i64: return fastEmit_X86ISD_VSRAI_MVT_v4i64_ri(RetVT, Op0, Op0IsKill, imm1);
14565 0 : case MVT::v8i64: return fastEmit_X86ISD_VSRAI_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
14566 : default: return 0;
14567 : }
14568 : }
14569 :
14570 : // FastEmit functions for X86ISD::VSRLDQ.
14571 :
14572 0 : unsigned fastEmit_X86ISD_VSRLDQ_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14573 0 : if (RetVT.SimpleTy != MVT::v16i8)
14574 : return 0;
14575 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14576 0 : return fastEmitInst_ri(X86::VPSRLDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14577 : }
14578 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14579 0 : return fastEmitInst_ri(X86::PSRLDQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14580 : }
14581 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14582 0 : return fastEmitInst_ri(X86::VPSRLDQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14583 : }
14584 : return 0;
14585 : }
14586 :
14587 0 : unsigned fastEmit_X86ISD_VSRLDQ_MVT_v32i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14588 0 : if (RetVT.SimpleTy != MVT::v32i8)
14589 : return 0;
14590 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14591 0 : return fastEmitInst_ri(X86::VPSRLDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14592 : }
14593 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14594 0 : return fastEmitInst_ri(X86::VPSRLDQYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14595 : }
14596 : return 0;
14597 : }
14598 :
14599 : unsigned fastEmit_X86ISD_VSRLDQ_MVT_v64i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14600 0 : if (RetVT.SimpleTy != MVT::v64i8)
14601 : return 0;
14602 0 : if ((Subtarget->hasBWI())) {
14603 0 : return fastEmitInst_ri(X86::VPSRLDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14604 : }
14605 : return 0;
14606 : }
14607 :
14608 0 : unsigned fastEmit_X86ISD_VSRLDQ_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14609 0 : switch (VT.SimpleTy) {
14610 0 : case MVT::v16i8: return fastEmit_X86ISD_VSRLDQ_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
14611 0 : case MVT::v32i8: return fastEmit_X86ISD_VSRLDQ_MVT_v32i8_ri(RetVT, Op0, Op0IsKill, imm1);
14612 0 : case MVT::v64i8: return fastEmit_X86ISD_VSRLDQ_MVT_v64i8_ri(RetVT, Op0, Op0IsKill, imm1);
14613 : default: return 0;
14614 : }
14615 : }
14616 :
14617 : // FastEmit functions for X86ISD::VSRLI.
14618 :
14619 0 : unsigned fastEmit_X86ISD_VSRLI_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14620 0 : if (RetVT.SimpleTy != MVT::v8i16)
14621 : return 0;
14622 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14623 0 : return fastEmitInst_ri(X86::VPSRLWZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14624 : }
14625 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14626 0 : return fastEmitInst_ri(X86::PSRLWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14627 : }
14628 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14629 0 : return fastEmitInst_ri(X86::VPSRLWri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14630 : }
14631 : return 0;
14632 : }
14633 :
14634 0 : unsigned fastEmit_X86ISD_VSRLI_MVT_v16i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14635 0 : if (RetVT.SimpleTy != MVT::v16i16)
14636 : return 0;
14637 0 : if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14638 0 : return fastEmitInst_ri(X86::VPSRLWZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14639 : }
14640 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14641 0 : return fastEmitInst_ri(X86::VPSRLWYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14642 : }
14643 : return 0;
14644 : }
14645 :
14646 : unsigned fastEmit_X86ISD_VSRLI_MVT_v32i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14647 0 : if (RetVT.SimpleTy != MVT::v32i16)
14648 : return 0;
14649 0 : if ((Subtarget->hasBWI())) {
14650 0 : return fastEmitInst_ri(X86::VPSRLWZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14651 : }
14652 : return 0;
14653 : }
14654 :
14655 0 : unsigned fastEmit_X86ISD_VSRLI_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14656 0 : if (RetVT.SimpleTy != MVT::v4i32)
14657 : return 0;
14658 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14659 0 : return fastEmitInst_ri(X86::VPSRLDZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14660 : }
14661 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14662 0 : return fastEmitInst_ri(X86::PSRLDri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14663 : }
14664 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14665 0 : return fastEmitInst_ri(X86::VPSRLDri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14666 : }
14667 : return 0;
14668 : }
14669 :
14670 0 : unsigned fastEmit_X86ISD_VSRLI_MVT_v8i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14671 0 : if (RetVT.SimpleTy != MVT::v8i32)
14672 : return 0;
14673 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14674 0 : return fastEmitInst_ri(X86::VPSRLDZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14675 : }
14676 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14677 0 : return fastEmitInst_ri(X86::VPSRLDYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14678 : }
14679 : return 0;
14680 : }
14681 :
14682 : unsigned fastEmit_X86ISD_VSRLI_MVT_v16i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14683 0 : if (RetVT.SimpleTy != MVT::v16i32)
14684 : return 0;
14685 0 : if ((Subtarget->hasAVX512())) {
14686 0 : return fastEmitInst_ri(X86::VPSRLDZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14687 : }
14688 : return 0;
14689 : }
14690 :
14691 0 : unsigned fastEmit_X86ISD_VSRLI_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14692 0 : if (RetVT.SimpleTy != MVT::v2i64)
14693 : return 0;
14694 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14695 0 : return fastEmitInst_ri(X86::VPSRLQZ128ri, &X86::VR128XRegClass, Op0, Op0IsKill, imm1);
14696 : }
14697 0 : if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14698 0 : return fastEmitInst_ri(X86::PSRLQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14699 : }
14700 0 : if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14701 0 : return fastEmitInst_ri(X86::VPSRLQri, &X86::VR128RegClass, Op0, Op0IsKill, imm1);
14702 : }
14703 : return 0;
14704 : }
14705 :
14706 0 : unsigned fastEmit_X86ISD_VSRLI_MVT_v4i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14707 0 : if (RetVT.SimpleTy != MVT::v4i64)
14708 : return 0;
14709 0 : if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14710 0 : return fastEmitInst_ri(X86::VPSRLQZ256ri, &X86::VR256XRegClass, Op0, Op0IsKill, imm1);
14711 : }
14712 0 : if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14713 0 : return fastEmitInst_ri(X86::VPSRLQYri, &X86::VR256RegClass, Op0, Op0IsKill, imm1);
14714 : }
14715 : return 0;
14716 : }
14717 :
14718 : unsigned fastEmit_X86ISD_VSRLI_MVT_v8i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14719 0 : if (RetVT.SimpleTy != MVT::v8i64)
14720 : return 0;
14721 0 : if ((Subtarget->hasAVX512())) {
14722 0 : return fastEmitInst_ri(X86::VPSRLQZri, &X86::VR512RegClass, Op0, Op0IsKill, imm1);
14723 : }
14724 : return 0;
14725 : }
14726 :
14727 0 : unsigned fastEmit_X86ISD_VSRLI_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14728 0 : switch (VT.SimpleTy) {
14729 0 : case MVT::v8i16: return fastEmit_X86ISD_VSRLI_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
14730 0 : case MVT::v16i16: return fastEmit_X86ISD_VSRLI_MVT_v16i16_ri(RetVT, Op0, Op0IsKill, imm1);
14731 0 : case MVT::v32i16: return fastEmit_X86ISD_VSRLI_MVT_v32i16_ri(RetVT, Op0, Op0IsKill, imm1);
14732 0 : case MVT::v4i32: return fastEmit_X86ISD_VSRLI_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
14733 0 : case MVT::v8i32: return fastEmit_X86ISD_VSRLI_MVT_v8i32_ri(RetVT, Op0, Op0IsKill, imm1);
14734 0 : case MVT::v16i32: return fastEmit_X86ISD_VSRLI_MVT_v16i32_ri(RetVT, Op0, Op0IsKill, imm1);
14735 0 : case MVT::v2i64: return fastEmit_X86ISD_VSRLI_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
14736 0 : case MVT::v4i64: return fastEmit_X86ISD_VSRLI_MVT_v4i64_ri(RetVT, Op0, Op0IsKill, imm1);
14737 0 : case MVT::v8i64: return fastEmit_X86ISD_VSRLI_MVT_v8i64_ri(RetVT, Op0, Op0IsKill, imm1);
14738 : default: return 0;
14739 : }
14740 : }
14741 :
14742 : // Top-level FastEmit function.
14743 :
14744 706351 : unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) override {
14745 706351 : if (VT == MVT::i16 && Predicate_i16immSExt8(imm1))
14746 13 : if (unsigned Reg = fastEmit_ri_Predicate_i16immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
14747 : return Reg;
14748 :
14749 706347 : if (VT == MVT::i32 && Predicate_i32immSExt8(imm1))
14750 68684 : if (unsigned Reg = fastEmit_ri_Predicate_i32immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
14751 : return Reg;
14752 :
14753 662676 : if (VT == MVT::i64 && Predicate_i64immSExt8(imm1))
14754 556599 : if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
14755 : return Reg;
14756 :
14757 217462 : if (VT == MVT::i64 && Predicate_i64immSExt32(imm1))
14758 124312 : if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
14759 : return Reg;
14760 :
14761 207368 : switch (Opcode) {
14762 122 : case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14763 44018 : case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14764 0 : case ISD::EXTRACT_SUBVECTOR: return fastEmit_ISD_EXTRACT_SUBVECTOR_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14765 0 : case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14766 1122 : case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14767 21 : case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14768 0 : case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14769 60088 : case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14770 52076 : case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14771 5612 : case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14772 44 : case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14773 21982 : case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14774 0 : case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14775 0 : case X86ISD::CVTP2SI_RND: return fastEmit_X86ISD_CVTP2SI_RND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14776 0 : case X86ISD::CVTP2UI_RND: return fastEmit_X86ISD_CVTP2UI_RND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14777 0 : case X86ISD::CVTPS2PH: return fastEmit_X86ISD_CVTPS2PH_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14778 0 : case X86ISD::CVTS2SI_RND: return fastEmit_X86ISD_CVTS2SI_RND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14779 0 : case X86ISD::CVTS2UI_RND: return fastEmit_X86ISD_CVTS2UI_RND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14780 0 : case X86ISD::FSQRT_RND: return fastEmit_X86ISD_FSQRT_RND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14781 0 : case X86ISD::KSHIFTL: return fastEmit_X86ISD_KSHIFTL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14782 0 : case X86ISD::KSHIFTR: return fastEmit_X86ISD_KSHIFTR_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14783 0 : case X86ISD::PEXTRB: return fastEmit_X86ISD_PEXTRB_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14784 0 : case X86ISD::PEXTRW: return fastEmit_X86ISD_PEXTRW_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14785 0 : case X86ISD::PSHUFD: return fastEmit_X86ISD_PSHUFD_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14786 0 : case X86ISD::PSHUFHW: return fastEmit_X86ISD_PSHUFHW_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14787 0 : case X86ISD::PSHUFLW: return fastEmit_X86ISD_PSHUFLW_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14788 0 : case X86ISD::SINT_TO_FP_RND: return fastEmit_X86ISD_SINT_TO_FP_RND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14789 0 : case X86ISD::UINT_TO_FP_RND: return fastEmit_X86ISD_UINT_TO_FP_RND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14790 0 : case X86ISD::VFPCLASS: return fastEmit_X86ISD_VFPCLASS_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14791 0 : case X86ISD::VFPCLASSS: return fastEmit_X86ISD_VFPCLASSS_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14792 0 : case X86ISD::VFPROUND_RND: return fastEmit_X86ISD_VFPROUND_RND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14793 0 : case X86ISD::VGETMANT: return fastEmit_X86ISD_VGETMANT_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14794 0 : case X86ISD::VPERMI: return fastEmit_X86ISD_VPERMI_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14795 0 : case X86ISD::VPERMILPI: return fastEmit_X86ISD_VPERMILPI_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14796 0 : case X86ISD::VREDUCE: return fastEmit_X86ISD_VREDUCE_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14797 0 : case X86ISD::VRNDSCALE: return fastEmit_X86ISD_VRNDSCALE_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14798 0 : case X86ISD::VROTLI: return fastEmit_X86ISD_VROTLI_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14799 0 : case X86ISD::VROTRI: return fastEmit_X86ISD_VROTRI_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14800 0 : case X86ISD::VSHLDQ: return fastEmit_X86ISD_VSHLDQ_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14801 0 : case X86ISD::VSHLI: return fastEmit_X86ISD_VSHLI_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14802 0 : case X86ISD::VSRAI: return fastEmit_X86ISD_VSRAI_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14803 0 : case X86ISD::VSRLDQ: return fastEmit_X86ISD_VSRLDQ_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14804 0 : case X86ISD::VSRLI: return fastEmit_X86ISD_VSRLI_ri(VT, RetVT, Op0, Op0IsKill, imm1);
14805 : default: return 0;
14806 : }
14807 : }
14808 :
14809 : // FastEmit functions for ISD::ADD.
14810 :
14811 : unsigned fastEmit_ISD_ADD_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14812 4 : if (RetVT.SimpleTy != MVT::i16)
14813 : return 0;
14814 4 : return fastEmitInst_ri(X86::ADD16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
14815 : }
14816 :
14817 : unsigned fastEmit_ISD_ADD_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14818 4 : switch (VT.SimpleTy) {
14819 4 : case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
14820 : default: return 0;
14821 : }
14822 : }
14823 :
14824 : // FastEmit functions for ISD::AND.
14825 :
14826 : unsigned fastEmit_ISD_AND_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14827 0 : if (RetVT.SimpleTy != MVT::i16)
14828 : return 0;
14829 0 : return fastEmitInst_ri(X86::AND16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
14830 : }
14831 :
14832 : unsigned fastEmit_ISD_AND_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14833 0 : switch (VT.SimpleTy) {
14834 0 : case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
14835 : default: return 0;
14836 : }
14837 : }
14838 :
14839 : // FastEmit functions for ISD::MUL.
14840 :
14841 : unsigned fastEmit_ISD_MUL_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14842 0 : if (RetVT.SimpleTy != MVT::i16)
14843 : return 0;
14844 0 : return fastEmitInst_ri(X86::IMUL16rri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
14845 : }
14846 :
14847 : unsigned fastEmit_ISD_MUL_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14848 0 : switch (VT.SimpleTy) {
14849 0 : case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
14850 : default: return 0;
14851 : }
14852 : }
14853 :
14854 : // FastEmit functions for ISD::OR.
14855 :
14856 : unsigned fastEmit_ISD_OR_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14857 0 : if (RetVT.SimpleTy != MVT::i16)
14858 : return 0;
14859 0 : return fastEmitInst_ri(X86::OR16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
14860 : }
14861 :
14862 : unsigned fastEmit_ISD_OR_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14863 0 : switch (VT.SimpleTy) {
14864 0 : case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
14865 : default: return 0;
14866 : }
14867 : }
14868 :
14869 : // FastEmit functions for ISD::SUB.
14870 :
14871 : unsigned fastEmit_ISD_SUB_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14872 0 : if (RetVT.SimpleTy != MVT::i16)
14873 : return 0;
14874 0 : return fastEmitInst_ri(X86::SUB16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
14875 : }
14876 :
14877 : unsigned fastEmit_ISD_SUB_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14878 0 : switch (VT.SimpleTy) {
14879 0 : case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
14880 : default: return 0;
14881 : }
14882 : }
14883 :
14884 : // FastEmit functions for ISD::XOR.
14885 :
14886 : unsigned fastEmit_ISD_XOR_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14887 0 : if (RetVT.SimpleTy != MVT::i16)
14888 : return 0;
14889 0 : return fastEmitInst_ri(X86::XOR16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
14890 : }
14891 :
14892 : unsigned fastEmit_ISD_XOR_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14893 0 : switch (VT.SimpleTy) {
14894 0 : case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
14895 : default: return 0;
14896 : }
14897 : }
14898 :
14899 : // FastEmit functions for X86ISD::BT.
14900 :
14901 : unsigned fastEmit_X86ISD_BT_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14902 0 : if (RetVT.SimpleTy != MVT::i32)
14903 : return 0;
14904 0 : return fastEmitInst_ri(X86::BT16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
14905 : }
14906 :
14907 : unsigned fastEmit_X86ISD_BT_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14908 0 : switch (VT.SimpleTy) {
14909 0 : case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
14910 : default: return 0;
14911 : }
14912 : }
14913 :
14914 : // Top-level FastEmit function.
14915 :
14916 13 : unsigned fastEmit_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14917 13 : switch (Opcode) {
14918 4 : case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
14919 0 : case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
14920 0 : case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
14921 0 : case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
14922 0 : case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
14923 0 : case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
14924 0 : case X86ISD::BT: return fastEmit_X86ISD_BT_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
14925 : default: return 0;
14926 : }
14927 : }
14928 :
14929 : // FastEmit functions for ISD::ADD.
14930 :
14931 : unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14932 25565 : if (RetVT.SimpleTy != MVT::i32)
14933 : return 0;
14934 25565 : return fastEmitInst_ri(X86::ADD32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
14935 : }
14936 :
14937 : unsigned fastEmit_ISD_ADD_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14938 25565 : switch (VT.SimpleTy) {
14939 25565 : case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
14940 : default: return 0;
14941 : }
14942 : }
14943 :
14944 : // FastEmit functions for ISD::AND.
14945 :
14946 : unsigned fastEmit_ISD_AND_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14947 7169 : if (RetVT.SimpleTy != MVT::i32)
14948 : return 0;
14949 7169 : return fastEmitInst_ri(X86::AND32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
14950 : }
14951 :
14952 : unsigned fastEmit_ISD_AND_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14953 7169 : switch (VT.SimpleTy) {
14954 7169 : case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
14955 : default: return 0;
14956 : }
14957 : }
14958 :
14959 : // FastEmit functions for ISD::MUL.
14960 :
14961 : unsigned fastEmit_ISD_MUL_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14962 278 : if (RetVT.SimpleTy != MVT::i32)
14963 : return 0;
14964 278 : return fastEmitInst_ri(X86::IMUL32rri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
14965 : }
14966 :
14967 : unsigned fastEmit_ISD_MUL_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14968 278 : switch (VT.SimpleTy) {
14969 278 : case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
14970 : default: return 0;
14971 : }
14972 : }
14973 :
14974 : // FastEmit functions for ISD::OR.
14975 :
14976 : unsigned fastEmit_ISD_OR_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14977 1045 : if (RetVT.SimpleTy != MVT::i32)
14978 : return 0;
14979 1045 : return fastEmitInst_ri(X86::OR32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
14980 : }
14981 :
14982 : unsigned fastEmit_ISD_OR_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14983 1045 : switch (VT.SimpleTy) {
14984 1045 : case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
14985 : default: return 0;
14986 : }
14987 : }
14988 :
14989 : // FastEmit functions for ISD::SUB.
14990 :
14991 : unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14992 4621 : if (RetVT.SimpleTy != MVT::i32)
14993 : return 0;
14994 4621 : return fastEmitInst_ri(X86::SUB32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
14995 : }
14996 :
14997 : unsigned fastEmit_ISD_SUB_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
14998 4621 : switch (VT.SimpleTy) {
14999 4621 : case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
15000 : default: return 0;
15001 : }
15002 : }
15003 :
15004 : // FastEmit functions for ISD::XOR.
15005 :
15006 : unsigned fastEmit_ISD_XOR_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15007 4993 : if (RetVT.SimpleTy != MVT::i32)
15008 : return 0;
15009 4993 : return fastEmitInst_ri(X86::XOR32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15010 : }
15011 :
15012 : unsigned fastEmit_ISD_XOR_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15013 4993 : switch (VT.SimpleTy) {
15014 4993 : case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
15015 : default: return 0;
15016 : }
15017 : }
15018 :
15019 : // FastEmit functions for X86ISD::BT.
15020 :
15021 : unsigned fastEmit_X86ISD_BT_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15022 0 : if (RetVT.SimpleTy != MVT::i32)
15023 : return 0;
15024 0 : return fastEmitInst_ri(X86::BT32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15025 : }
15026 :
15027 : unsigned fastEmit_X86ISD_BT_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15028 0 : switch (VT.SimpleTy) {
15029 0 : case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
15030 : default: return 0;
15031 : }
15032 : }
15033 :
15034 : // Top-level FastEmit function.
15035 :
15036 68684 : unsigned fastEmit_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15037 68684 : switch (Opcode) {
15038 25565 : case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15039 7169 : case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15040 278 : case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15041 1045 : case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15042 4621 : case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15043 4993 : case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15044 0 : case X86ISD::BT: return fastEmit_X86ISD_BT_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15045 : default: return 0;
15046 : }
15047 : }
15048 :
15049 : // FastEmit functions for ISD::ADD.
15050 :
15051 : unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15052 357409 : if (RetVT.SimpleTy != MVT::i64)
15053 : return 0;
15054 357409 : return fastEmitInst_ri(X86::ADD64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15055 : }
15056 :
15057 : unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15058 357409 : switch (VT.SimpleTy) {
15059 357409 : case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
15060 : default: return 0;
15061 : }
15062 : }
15063 :
15064 : // FastEmit functions for ISD::AND.
15065 :
15066 : unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15067 44877 : if (RetVT.SimpleTy != MVT::i64)
15068 : return 0;
15069 44877 : return fastEmitInst_ri(X86::AND64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15070 : }
15071 :
15072 : unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15073 44877 : switch (VT.SimpleTy) {
15074 44877 : case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
15075 : default: return 0;
15076 : }
15077 : }
15078 :
15079 : // FastEmit functions for ISD::MUL.
15080 :
15081 : unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15082 15462 : if (RetVT.SimpleTy != MVT::i64)
15083 : return 0;
15084 15462 : return fastEmitInst_ri(X86::IMUL64rri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15085 : }
15086 :
15087 : unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15088 15462 : switch (VT.SimpleTy) {
15089 15462 : case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
15090 : default: return 0;
15091 : }
15092 : }
15093 :
15094 : // FastEmit functions for ISD::OR.
15095 :
15096 : unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15097 1799 : if (RetVT.SimpleTy != MVT::i64)
15098 : return 0;
15099 1799 : return fastEmitInst_ri(X86::OR64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15100 : }
15101 :
15102 : unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15103 1799 : switch (VT.SimpleTy) {
15104 1799 : case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
15105 : default: return 0;
15106 : }
15107 : }
15108 :
15109 : // FastEmit functions for ISD::SUB.
15110 :
15111 : unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15112 23873 : if (RetVT.SimpleTy != MVT::i64)
15113 : return 0;
15114 23873 : return fastEmitInst_ri(X86::SUB64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15115 : }
15116 :
15117 : unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15118 23873 : switch (VT.SimpleTy) {
15119 23873 : case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
15120 : default: return 0;
15121 : }
15122 : }
15123 :
15124 : // FastEmit functions for ISD::XOR.
15125 :
15126 : unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15127 1794 : if (RetVT.SimpleTy != MVT::i64)
15128 : return 0;
15129 1794 : return fastEmitInst_ri(X86::XOR64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15130 : }
15131 :
15132 : unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15133 1794 : switch (VT.SimpleTy) {
15134 1794 : case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
15135 : default: return 0;
15136 : }
15137 : }
15138 :
15139 : // FastEmit functions for X86ISD::BT.
15140 :
15141 : unsigned fastEmit_X86ISD_BT_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15142 0 : if (RetVT.SimpleTy != MVT::i32)
15143 : return 0;
15144 0 : return fastEmitInst_ri(X86::BT64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15145 : }
15146 :
15147 : unsigned fastEmit_X86ISD_BT_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15148 0 : switch (VT.SimpleTy) {
15149 0 : case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
15150 : default: return 0;
15151 : }
15152 : }
15153 :
15154 : // Top-level FastEmit function.
15155 :
15156 556599 : unsigned fastEmit_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15157 556599 : switch (Opcode) {
15158 357409 : case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15159 44877 : case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15160 15462 : case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15161 1799 : case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15162 23873 : case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15163 1794 : case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15164 0 : case X86ISD::BT: return fastEmit_X86ISD_BT_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15165 : default: return 0;
15166 : }
15167 : }
15168 :
15169 : // FastEmit functions for ISD::ADD.
15170 :
15171 : unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15172 7815 : if (RetVT.SimpleTy != MVT::i64)
15173 : return 0;
15174 7815 : return fastEmitInst_ri(X86::ADD64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15175 : }
15176 :
15177 : unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15178 7815 : switch (VT.SimpleTy) {
15179 7815 : case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
15180 : default: return 0;
15181 : }
15182 : }
15183 :
15184 : // FastEmit functions for ISD::AND.
15185 :
15186 : unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15187 974 : if (RetVT.SimpleTy != MVT::i64)
15188 : return 0;
15189 974 : return fastEmitInst_ri(X86::AND64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15190 : }
15191 :
15192 : unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15193 974 : switch (VT.SimpleTy) {
15194 974 : case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
15195 : default: return 0;
15196 : }
15197 : }
15198 :
15199 : // FastEmit functions for ISD::MUL.
15200 :
15201 : unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15202 965 : if (RetVT.SimpleTy != MVT::i64)
15203 : return 0;
15204 965 : return fastEmitInst_ri(X86::IMUL64rri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15205 : }
15206 :
15207 : unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15208 965 : switch (VT.SimpleTy) {
15209 965 : case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
15210 : default: return 0;
15211 : }
15212 : }
15213 :
15214 : // FastEmit functions for ISD::OR.
15215 :
15216 : unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15217 0 : if (RetVT.SimpleTy != MVT::i64)
15218 : return 0;
15219 0 : return fastEmitInst_ri(X86::OR64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15220 : }
15221 :
15222 : unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15223 0 : switch (VT.SimpleTy) {
15224 0 : case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
15225 : default: return 0;
15226 : }
15227 : }
15228 :
15229 : // FastEmit functions for ISD::SUB.
15230 :
15231 : unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15232 340 : if (RetVT.SimpleTy != MVT::i64)
15233 : return 0;
15234 340 : return fastEmitInst_ri(X86::SUB64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15235 : }
15236 :
15237 : unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15238 340 : switch (VT.SimpleTy) {
15239 340 : case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
15240 : default: return 0;
15241 : }
15242 : }
15243 :
15244 : // FastEmit functions for ISD::XOR.
15245 :
15246 : unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15247 0 : if (RetVT.SimpleTy != MVT::i64)
15248 : return 0;
15249 0 : return fastEmitInst_ri(X86::XOR64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15250 : }
15251 :
15252 : unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15253 0 : switch (VT.SimpleTy) {
15254 0 : case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
15255 : default: return 0;
15256 : }
15257 : }
15258 :
15259 : // FastEmit functions for X86ISD::BEXTR.
15260 :
15261 : unsigned fastEmit_X86ISD_BEXTR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15262 0 : if (RetVT.SimpleTy != MVT::i64)
15263 : return 0;
15264 0 : if ((Subtarget->hasTBM())) {
15265 0 : return fastEmitInst_ri(X86::BEXTRI64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15266 : }
15267 : return 0;
15268 : }
15269 :
15270 0 : unsigned fastEmit_X86ISD_BEXTR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15271 0 : switch (VT.SimpleTy) {
15272 0 : case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
15273 : default: return 0;
15274 : }
15275 : }
15276 :
15277 : // Top-level FastEmit function.
15278 :
15279 124312 : unsigned fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15280 124312 : switch (Opcode) {
15281 7815 : case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
15282 974 : case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
15283 965 : case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
15284 0 : case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
15285 340 : case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
15286 0 : case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
15287 0 : case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
15288 : default: return 0;
15289 : }
15290 : }
15291 :
15292 : // FastEmit functions for ISD::Constant.
15293 :
15294 : unsigned fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
15295 5 : if (RetVT.SimpleTy != MVT::i8)
15296 : return 0;
15297 5 : return fastEmitInst_i(X86::MOV8ri, &X86::GR8RegClass, imm0);
15298 : }
15299 :
15300 : unsigned fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
15301 0 : if (RetVT.SimpleTy != MVT::i16)
15302 : return 0;
15303 0 : return fastEmitInst_i(X86::MOV16ri, &X86::GR16RegClass, imm0);
15304 : }
15305 :
15306 22308 : unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
15307 22308 : switch (VT.SimpleTy) {
15308 5 : case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
15309 0 : case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
15310 : default: return 0;
15311 : }
15312 : }
15313 :
15314 : // Top-level FastEmit function.
15315 :
15316 22308 : unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
15317 22308 : switch (Opcode) {
15318 22308 : case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
15319 : default: return 0;
15320 : }
15321 : }
15322 :
|
