LLVM 19.0.0git
X86FastISel.cpp
Go to the documentation of this file.
1//===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the X86-specific support for the FastISel class. Much
10// of the target-specific code is generated by tablegen in the file
11// X86GenFastISel.inc, which is #included here.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86CallingConv.h"
17#include "X86InstrBuilder.h"
18#include "X86InstrInfo.h"
20#include "X86RegisterInfo.h"
21#include "X86Subtarget.h"
22#include "X86TargetMachine.h"
29#include "llvm/IR/CallingConv.h"
30#include "llvm/IR/DebugInfo.h"
33#include "llvm/IR/GlobalAlias.h"
37#include "llvm/IR/IntrinsicsX86.h"
38#include "llvm/IR/Operator.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCSymbol.h"
43using namespace llvm;
44
45namespace {
46
47class X86FastISel final : public FastISel {
48 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
49 /// make the right decision when generating code for different targets.
50 const X86Subtarget *Subtarget;
51
52public:
53 explicit X86FastISel(FunctionLoweringInfo &funcInfo,
54 const TargetLibraryInfo *libInfo)
55 : FastISel(funcInfo, libInfo) {
56 Subtarget = &funcInfo.MF->getSubtarget<X86Subtarget>();
57 }
58
59 bool fastSelectInstruction(const Instruction *I) override;
60
61 /// The specified machine instr operand is a vreg, and that
62 /// vreg is being provided by the specified load instruction. If possible,
63 /// try to fold the load as an operand to the instruction, returning true if
64 /// possible.
65 bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
66 const LoadInst *LI) override;
67
68 bool fastLowerArguments() override;
69 bool fastLowerCall(CallLoweringInfo &CLI) override;
70 bool fastLowerIntrinsicCall(const IntrinsicInst *II) override;
71
72#include "X86GenFastISel.inc"
73
74private:
75 bool X86FastEmitCompare(const Value *LHS, const Value *RHS, EVT VT,
76 const DebugLoc &DL);
77
78 bool X86FastEmitLoad(MVT VT, X86AddressMode &AM, MachineMemOperand *MMO,
79 unsigned &ResultReg, unsigned Alignment = 1);
80
81 bool X86FastEmitStore(EVT VT, const Value *Val, X86AddressMode &AM,
82 MachineMemOperand *MMO = nullptr, bool Aligned = false);
83 bool X86FastEmitStore(EVT VT, unsigned ValReg, X86AddressMode &AM,
84 MachineMemOperand *MMO = nullptr, bool Aligned = false);
85
86 bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, unsigned Src, EVT SrcVT,
87 unsigned &ResultReg);
88
89 bool X86SelectAddress(const Value *V, X86AddressMode &AM);
90 bool X86SelectCallAddress(const Value *V, X86AddressMode &AM);
91
92 bool X86SelectLoad(const Instruction *I);
93
94 bool X86SelectStore(const Instruction *I);
95
96 bool X86SelectRet(const Instruction *I);
97
98 bool X86SelectCmp(const Instruction *I);
99
100 bool X86SelectZExt(const Instruction *I);
101
102 bool X86SelectSExt(const Instruction *I);
103
104 bool X86SelectBranch(const Instruction *I);
105
106 bool X86SelectShift(const Instruction *I);
107
108 bool X86SelectDivRem(const Instruction *I);
109
110 bool X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I);
111
112 bool X86FastEmitSSESelect(MVT RetVT, const Instruction *I);
113
114 bool X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I);
115
116 bool X86SelectSelect(const Instruction *I);
117
118 bool X86SelectTrunc(const Instruction *I);
119
120 bool X86SelectFPExtOrFPTrunc(const Instruction *I, unsigned Opc,
121 const TargetRegisterClass *RC);
122
123 bool X86SelectFPExt(const Instruction *I);
124 bool X86SelectFPTrunc(const Instruction *I);
125 bool X86SelectSIToFP(const Instruction *I);
126 bool X86SelectUIToFP(const Instruction *I);
127 bool X86SelectIntToFP(const Instruction *I, bool IsSigned);
128
129 const X86InstrInfo *getInstrInfo() const {
130 return Subtarget->getInstrInfo();
131 }
132 const X86TargetMachine *getTargetMachine() const {
133 return static_cast<const X86TargetMachine *>(&TM);
134 }
135
136 bool handleConstantAddresses(const Value *V, X86AddressMode &AM);
137
138 unsigned X86MaterializeInt(const ConstantInt *CI, MVT VT);
139 unsigned X86MaterializeFP(const ConstantFP *CFP, MVT VT);
140 unsigned X86MaterializeGV(const GlobalValue *GV, MVT VT);
141 unsigned fastMaterializeConstant(const Constant *C) override;
142
143 unsigned fastMaterializeAlloca(const AllocaInst *C) override;
144
145 unsigned fastMaterializeFloatZero(const ConstantFP *CF) override;
146
147 /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is
148 /// computed in an SSE register, not on the X87 floating point stack.
149 bool isScalarFPTypeInSSEReg(EVT VT) const {
150 return (VT == MVT::f64 && Subtarget->hasSSE2()) ||
151 (VT == MVT::f32 && Subtarget->hasSSE1()) || VT == MVT::f16;
152 }
153
154 bool isTypeLegal(Type *Ty, MVT &VT, bool AllowI1 = false);
155
156 bool IsMemcpySmall(uint64_t Len);
157
158 bool TryEmitSmallMemcpy(X86AddressMode DestAM,
159 X86AddressMode SrcAM, uint64_t Len);
160
161 bool foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
162 const Value *Cond);
163
165 X86AddressMode &AM);
166
167 unsigned fastEmitInst_rrrr(unsigned MachineInstOpcode,
168 const TargetRegisterClass *RC, unsigned Op0,
169 unsigned Op1, unsigned Op2, unsigned Op3);
170};
171
172} // end anonymous namespace.
173
174static std::pair<unsigned, bool>
176 unsigned CC;
177 bool NeedSwap = false;
178
179 // SSE Condition code mapping:
180 // 0 - EQ
181 // 1 - LT
182 // 2 - LE
183 // 3 - UNORD
184 // 4 - NEQ
185 // 5 - NLT
186 // 6 - NLE
187 // 7 - ORD
188 switch (Predicate) {
189 default: llvm_unreachable("Unexpected predicate");
190 case CmpInst::FCMP_OEQ: CC = 0; break;
191 case CmpInst::FCMP_OGT: NeedSwap = true; [[fallthrough]];
192 case CmpInst::FCMP_OLT: CC = 1; break;
193 case CmpInst::FCMP_OGE: NeedSwap = true; [[fallthrough]];
194 case CmpInst::FCMP_OLE: CC = 2; break;
195 case CmpInst::FCMP_UNO: CC = 3; break;
196 case CmpInst::FCMP_UNE: CC = 4; break;
197 case CmpInst::FCMP_ULE: NeedSwap = true; [[fallthrough]];
198 case CmpInst::FCMP_UGE: CC = 5; break;
199 case CmpInst::FCMP_ULT: NeedSwap = true; [[fallthrough]];
200 case CmpInst::FCMP_UGT: CC = 6; break;
201 case CmpInst::FCMP_ORD: CC = 7; break;
202 case CmpInst::FCMP_UEQ: CC = 8; break;
203 case CmpInst::FCMP_ONE: CC = 12; break;
204 }
205
206 return std::make_pair(CC, NeedSwap);
207}
208
209/// Adds a complex addressing mode to the given machine instr builder.
210/// Note, this will constrain the index register. If its not possible to
211/// constrain the given index register, then a new one will be created. The
212/// IndexReg field of the addressing mode will be updated to match in this case.
214X86FastISel::addFullAddress(const MachineInstrBuilder &MIB,
215 X86AddressMode &AM) {
216 // First constrain the index register. It needs to be a GR64_NOSP.
218 MIB->getNumOperands() +
220 return ::addFullAddress(MIB, AM);
221}
222
223/// Check if it is possible to fold the condition from the XALU intrinsic
224/// into the user. The condition code will only be updated on success.
225bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
226 const Value *Cond) {
227 if (!isa<ExtractValueInst>(Cond))
228 return false;
229
230 const auto *EV = cast<ExtractValueInst>(Cond);
231 if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
232 return false;
233
234 const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());
235 MVT RetVT;
236 const Function *Callee = II->getCalledFunction();
237 Type *RetTy =
238 cast<StructType>(Callee->getReturnType())->getTypeAtIndex(0U);
239 if (!isTypeLegal(RetTy, RetVT))
240 return false;
241
242 if (RetVT != MVT::i32 && RetVT != MVT::i64)
243 return false;
244
245 X86::CondCode TmpCC;
246 switch (II->getIntrinsicID()) {
247 default: return false;
248 case Intrinsic::sadd_with_overflow:
249 case Intrinsic::ssub_with_overflow:
250 case Intrinsic::smul_with_overflow:
251 case Intrinsic::umul_with_overflow: TmpCC = X86::COND_O; break;
252 case Intrinsic::uadd_with_overflow:
253 case Intrinsic::usub_with_overflow: TmpCC = X86::COND_B; break;
254 }
255
256 // Check if both instructions are in the same basic block.
257 if (II->getParent() != I->getParent())
258 return false;
259
260 // Make sure nothing is in the way
263 for (auto Itr = std::prev(Start); Itr != End; --Itr) {
264 // We only expect extractvalue instructions between the intrinsic and the
265 // instruction to be selected.
266 if (!isa<ExtractValueInst>(Itr))
267 return false;
268
269 // Check that the extractvalue operand comes from the intrinsic.
270 const auto *EVI = cast<ExtractValueInst>(Itr);
271 if (EVI->getAggregateOperand() != II)
272 return false;
273 }
274
275 // Make sure no potentially eflags clobbering phi moves can be inserted in
276 // between.
277 auto HasPhis = [](const BasicBlock *Succ) { return !Succ->phis().empty(); };
278 if (I->isTerminator() && llvm::any_of(successors(I), HasPhis))
279 return false;
280
281 // Make sure there are no potentially eflags clobbering constant
282 // materializations in between.
283 if (llvm::any_of(I->operands(), [](Value *V) { return isa<Constant>(V); }))
284 return false;
285
286 CC = TmpCC;
287 return true;
288}
289
290bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
291 EVT evt = TLI.getValueType(DL, Ty, /*AllowUnknown=*/true);
292 if (evt == MVT::Other || !evt.isSimple())
293 // Unhandled type. Halt "fast" selection and bail.
294 return false;
295
296 VT = evt.getSimpleVT();
297 // For now, require SSE/SSE2 for performing floating-point operations,
298 // since x87 requires additional work.
299 if (VT == MVT::f64 && !Subtarget->hasSSE2())
300 return false;
301 if (VT == MVT::f32 && !Subtarget->hasSSE1())
302 return false;
303 // Similarly, no f80 support yet.
304 if (VT == MVT::f80)
305 return false;
306 // We only handle legal types. For example, on x86-32 the instruction
307 // selector contains all of the 64-bit instructions from x86-64,
308 // under the assumption that i64 won't be used if the target doesn't
309 // support it.
310 return (AllowI1 && VT == MVT::i1) || TLI.isTypeLegal(VT);
311}
312
313/// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
314/// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
315/// Return true and the result register by reference if it is possible.
316bool X86FastISel::X86FastEmitLoad(MVT VT, X86AddressMode &AM,
317 MachineMemOperand *MMO, unsigned &ResultReg,
318 unsigned Alignment) {
319 bool HasSSE1 = Subtarget->hasSSE1();
320 bool HasSSE2 = Subtarget->hasSSE2();
321 bool HasSSE41 = Subtarget->hasSSE41();
322 bool HasAVX = Subtarget->hasAVX();
323 bool HasAVX2 = Subtarget->hasAVX2();
324 bool HasAVX512 = Subtarget->hasAVX512();
325 bool HasVLX = Subtarget->hasVLX();
326 bool IsNonTemporal = MMO && MMO->isNonTemporal();
327
328 // Treat i1 loads the same as i8 loads. Masking will be done when storing.
329 if (VT == MVT::i1)
330 VT = MVT::i8;
331
332 // Get opcode and regclass of the output for the given load instruction.
333 unsigned Opc = 0;
334 switch (VT.SimpleTy) {
335 default: return false;
336 case MVT::i8:
337 Opc = X86::MOV8rm;
338 break;
339 case MVT::i16:
340 Opc = X86::MOV16rm;
341 break;
342 case MVT::i32:
343 Opc = X86::MOV32rm;
344 break;
345 case MVT::i64:
346 // Must be in x86-64 mode.
347 Opc = X86::MOV64rm;
348 break;
349 case MVT::f32:
350 Opc = HasAVX512 ? X86::VMOVSSZrm_alt
351 : HasAVX ? X86::VMOVSSrm_alt
352 : HasSSE1 ? X86::MOVSSrm_alt
353 : X86::LD_Fp32m;
354 break;
355 case MVT::f64:
356 Opc = HasAVX512 ? X86::VMOVSDZrm_alt
357 : HasAVX ? X86::VMOVSDrm_alt
358 : HasSSE2 ? X86::MOVSDrm_alt
359 : X86::LD_Fp64m;
360 break;
361 case MVT::f80:
362 // No f80 support yet.
363 return false;
364 case MVT::v4f32:
365 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
366 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
367 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
368 else if (Alignment >= 16)
369 Opc = HasVLX ? X86::VMOVAPSZ128rm :
370 HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm;
371 else
372 Opc = HasVLX ? X86::VMOVUPSZ128rm :
373 HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm;
374 break;
375 case MVT::v2f64:
376 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
377 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
378 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
379 else if (Alignment >= 16)
380 Opc = HasVLX ? X86::VMOVAPDZ128rm :
381 HasAVX ? X86::VMOVAPDrm : X86::MOVAPDrm;
382 else
383 Opc = HasVLX ? X86::VMOVUPDZ128rm :
384 HasAVX ? X86::VMOVUPDrm : X86::MOVUPDrm;
385 break;
386 case MVT::v4i32:
387 case MVT::v2i64:
388 case MVT::v8i16:
389 case MVT::v16i8:
390 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
391 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
392 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
393 else if (Alignment >= 16)
394 Opc = HasVLX ? X86::VMOVDQA64Z128rm :
395 HasAVX ? X86::VMOVDQArm : X86::MOVDQArm;
396 else
397 Opc = HasVLX ? X86::VMOVDQU64Z128rm :
398 HasAVX ? X86::VMOVDQUrm : X86::MOVDQUrm;
399 break;
400 case MVT::v8f32:
401 assert(HasAVX);
402 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
403 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
404 else if (IsNonTemporal && Alignment >= 16)
405 return false; // Force split for X86::VMOVNTDQArm
406 else if (Alignment >= 32)
407 Opc = HasVLX ? X86::VMOVAPSZ256rm : X86::VMOVAPSYrm;
408 else
409 Opc = HasVLX ? X86::VMOVUPSZ256rm : X86::VMOVUPSYrm;
410 break;
411 case MVT::v4f64:
412 assert(HasAVX);
413 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
414 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
415 else if (IsNonTemporal && Alignment >= 16)
416 return false; // Force split for X86::VMOVNTDQArm
417 else if (Alignment >= 32)
418 Opc = HasVLX ? X86::VMOVAPDZ256rm : X86::VMOVAPDYrm;
419 else
420 Opc = HasVLX ? X86::VMOVUPDZ256rm : X86::VMOVUPDYrm;
421 break;
422 case MVT::v8i32:
423 case MVT::v4i64:
424 case MVT::v16i16:
425 case MVT::v32i8:
426 assert(HasAVX);
427 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
428 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
429 else if (IsNonTemporal && Alignment >= 16)
430 return false; // Force split for X86::VMOVNTDQArm
431 else if (Alignment >= 32)
432 Opc = HasVLX ? X86::VMOVDQA64Z256rm : X86::VMOVDQAYrm;
433 else
434 Opc = HasVLX ? X86::VMOVDQU64Z256rm : X86::VMOVDQUYrm;
435 break;
436 case MVT::v16f32:
437 assert(HasAVX512);
438 if (IsNonTemporal && Alignment >= 64)
439 Opc = X86::VMOVNTDQAZrm;
440 else
441 Opc = (Alignment >= 64) ? X86::VMOVAPSZrm : X86::VMOVUPSZrm;
442 break;
443 case MVT::v8f64:
444 assert(HasAVX512);
445 if (IsNonTemporal && Alignment >= 64)
446 Opc = X86::VMOVNTDQAZrm;
447 else
448 Opc = (Alignment >= 64) ? X86::VMOVAPDZrm : X86::VMOVUPDZrm;
449 break;
450 case MVT::v8i64:
451 case MVT::v16i32:
452 case MVT::v32i16:
453 case MVT::v64i8:
454 assert(HasAVX512);
455 // Note: There are a lot more choices based on type with AVX-512, but
456 // there's really no advantage when the load isn't masked.
457 if (IsNonTemporal && Alignment >= 64)
458 Opc = X86::VMOVNTDQAZrm;
459 else
460 Opc = (Alignment >= 64) ? X86::VMOVDQA64Zrm : X86::VMOVDQU64Zrm;
461 break;
462 }
463
464 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
465
466 ResultReg = createResultReg(RC);
468 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg);
469 addFullAddress(MIB, AM);
470 if (MMO)
471 MIB->addMemOperand(*FuncInfo.MF, MMO);
472 return true;
473}
474
475/// X86FastEmitStore - Emit a machine instruction to store a value Val of
476/// type VT. The address is either pre-computed, consisted of a base ptr, Ptr
477/// and a displacement offset, or a GlobalAddress,
478/// i.e. V. Return true if it is possible.
479bool X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, X86AddressMode &AM,
480 MachineMemOperand *MMO, bool Aligned) {
481 bool HasSSE1 = Subtarget->hasSSE1();
482 bool HasSSE2 = Subtarget->hasSSE2();
483 bool HasSSE4A = Subtarget->hasSSE4A();
484 bool HasAVX = Subtarget->hasAVX();
485 bool HasAVX512 = Subtarget->hasAVX512();
486 bool HasVLX = Subtarget->hasVLX();
487 bool IsNonTemporal = MMO && MMO->isNonTemporal();
488
489 // Get opcode and regclass of the output for the given store instruction.
490 unsigned Opc = 0;
491 switch (VT.getSimpleVT().SimpleTy) {
492 case MVT::f80: // No f80 support yet.
493 default: return false;
494 case MVT::i1: {
495 // Mask out all but lowest bit.
496 Register AndResult = createResultReg(&X86::GR8RegClass);
497 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
498 TII.get(X86::AND8ri), AndResult)
499 .addReg(ValReg).addImm(1);
500 ValReg = AndResult;
501 [[fallthrough]]; // handle i1 as i8.
502 }
503 case MVT::i8: Opc = X86::MOV8mr; break;
504 case MVT::i16: Opc = X86::MOV16mr; break;
505 case MVT::i32:
506 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTImr : X86::MOV32mr;
507 break;
508 case MVT::i64:
509 // Must be in x86-64 mode.
510 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTI_64mr : X86::MOV64mr;
511 break;
512 case MVT::f32:
513 if (HasSSE1) {
514 if (IsNonTemporal && HasSSE4A)
515 Opc = X86::MOVNTSS;
516 else
517 Opc = HasAVX512 ? X86::VMOVSSZmr :
518 HasAVX ? X86::VMOVSSmr : X86::MOVSSmr;
519 } else
520 Opc = X86::ST_Fp32m;
521 break;
522 case MVT::f64:
523 if (HasSSE2) {
524 if (IsNonTemporal && HasSSE4A)
525 Opc = X86::MOVNTSD;
526 else
527 Opc = HasAVX512 ? X86::VMOVSDZmr :
528 HasAVX ? X86::VMOVSDmr : X86::MOVSDmr;
529 } else
530 Opc = X86::ST_Fp64m;
531 break;
532 case MVT::x86mmx:
533 Opc = (IsNonTemporal && HasSSE1) ? X86::MMX_MOVNTQmr : X86::MMX_MOVQ64mr;
534 break;
535 case MVT::v4f32:
536 if (Aligned) {
537 if (IsNonTemporal)
538 Opc = HasVLX ? X86::VMOVNTPSZ128mr :
539 HasAVX ? X86::VMOVNTPSmr : X86::MOVNTPSmr;
540 else
541 Opc = HasVLX ? X86::VMOVAPSZ128mr :
542 HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr;
543 } else
544 Opc = HasVLX ? X86::VMOVUPSZ128mr :
545 HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr;
546 break;
547 case MVT::v2f64:
548 if (Aligned) {
549 if (IsNonTemporal)
550 Opc = HasVLX ? X86::VMOVNTPDZ128mr :
551 HasAVX ? X86::VMOVNTPDmr : X86::MOVNTPDmr;
552 else
553 Opc = HasVLX ? X86::VMOVAPDZ128mr :
554 HasAVX ? X86::VMOVAPDmr : X86::MOVAPDmr;
555 } else
556 Opc = HasVLX ? X86::VMOVUPDZ128mr :
557 HasAVX ? X86::VMOVUPDmr : X86::MOVUPDmr;
558 break;
559 case MVT::v4i32:
560 case MVT::v2i64:
561 case MVT::v8i16:
562 case MVT::v16i8:
563 if (Aligned) {
564 if (IsNonTemporal)
565 Opc = HasVLX ? X86::VMOVNTDQZ128mr :
566 HasAVX ? X86::VMOVNTDQmr : X86::MOVNTDQmr;
567 else
568 Opc = HasVLX ? X86::VMOVDQA64Z128mr :
569 HasAVX ? X86::VMOVDQAmr : X86::MOVDQAmr;
570 } else
571 Opc = HasVLX ? X86::VMOVDQU64Z128mr :
572 HasAVX ? X86::VMOVDQUmr : X86::MOVDQUmr;
573 break;
574 case MVT::v8f32:
575 assert(HasAVX);
576 if (Aligned) {
577 if (IsNonTemporal)
578 Opc = HasVLX ? X86::VMOVNTPSZ256mr : X86::VMOVNTPSYmr;
579 else
580 Opc = HasVLX ? X86::VMOVAPSZ256mr : X86::VMOVAPSYmr;
581 } else
582 Opc = HasVLX ? X86::VMOVUPSZ256mr : X86::VMOVUPSYmr;
583 break;
584 case MVT::v4f64:
585 assert(HasAVX);
586 if (Aligned) {
587 if (IsNonTemporal)
588 Opc = HasVLX ? X86::VMOVNTPDZ256mr : X86::VMOVNTPDYmr;
589 else
590 Opc = HasVLX ? X86::VMOVAPDZ256mr : X86::VMOVAPDYmr;
591 } else
592 Opc = HasVLX ? X86::VMOVUPDZ256mr : X86::VMOVUPDYmr;
593 break;
594 case MVT::v8i32:
595 case MVT::v4i64:
596 case MVT::v16i16:
597 case MVT::v32i8:
598 assert(HasAVX);
599 if (Aligned) {
600 if (IsNonTemporal)
601 Opc = HasVLX ? X86::VMOVNTDQZ256mr : X86::VMOVNTDQYmr;
602 else
603 Opc = HasVLX ? X86::VMOVDQA64Z256mr : X86::VMOVDQAYmr;
604 } else
605 Opc = HasVLX ? X86::VMOVDQU64Z256mr : X86::VMOVDQUYmr;
606 break;
607 case MVT::v16f32:
608 assert(HasAVX512);
609 if (Aligned)
610 Opc = IsNonTemporal ? X86::VMOVNTPSZmr : X86::VMOVAPSZmr;
611 else
612 Opc = X86::VMOVUPSZmr;
613 break;
614 case MVT::v8f64:
615 assert(HasAVX512);
616 if (Aligned) {
617 Opc = IsNonTemporal ? X86::VMOVNTPDZmr : X86::VMOVAPDZmr;
618 } else
619 Opc = X86::VMOVUPDZmr;
620 break;
621 case MVT::v8i64:
622 case MVT::v16i32:
623 case MVT::v32i16:
624 case MVT::v64i8:
625 assert(HasAVX512);
626 // Note: There are a lot more choices based on type with AVX-512, but
627 // there's really no advantage when the store isn't masked.
628 if (Aligned)
629 Opc = IsNonTemporal ? X86::VMOVNTDQZmr : X86::VMOVDQA64Zmr;
630 else
631 Opc = X86::VMOVDQU64Zmr;
632 break;
633 }
634
635 const MCInstrDesc &Desc = TII.get(Opc);
636 // Some of the instructions in the previous switch use FR128 instead
637 // of FR32 for ValReg. Make sure the register we feed the instruction
638 // matches its register class constraints.
639 // Note: This is fine to do a copy from FR32 to FR128, this is the
640 // same registers behind the scene and actually why it did not trigger
641 // any bugs before.
642 ValReg = constrainOperandRegClass(Desc, ValReg, Desc.getNumOperands() - 1);
644 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, Desc);
645 addFullAddress(MIB, AM).addReg(ValReg);
646 if (MMO)
647 MIB->addMemOperand(*FuncInfo.MF, MMO);
648
649 return true;
650}
651
652bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
653 X86AddressMode &AM,
654 MachineMemOperand *MMO, bool Aligned) {
655 // Handle 'null' like i32/i64 0.
656 if (isa<ConstantPointerNull>(Val))
657 Val = Constant::getNullValue(DL.getIntPtrType(Val->getContext()));
658
659 // If this is a store of a simple constant, fold the constant into the store.
660 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
661 unsigned Opc = 0;
662 bool Signed = true;
663 switch (VT.getSimpleVT().SimpleTy) {
664 default: break;
665 case MVT::i1:
666 Signed = false;
667 [[fallthrough]]; // Handle as i8.
668 case MVT::i8: Opc = X86::MOV8mi; break;
669 case MVT::i16: Opc = X86::MOV16mi; break;
670 case MVT::i32: Opc = X86::MOV32mi; break;
671 case MVT::i64:
672 // Must be a 32-bit sign extended value.
673 if (isInt<32>(CI->getSExtValue()))
674 Opc = X86::MOV64mi32;
675 break;
676 }
677
678 if (Opc) {
680 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc));
681 addFullAddress(MIB, AM).addImm(Signed ? (uint64_t) CI->getSExtValue()
682 : CI->getZExtValue());
683 if (MMO)
684 MIB->addMemOperand(*FuncInfo.MF, MMO);
685 return true;
686 }
687 }
688
689 Register ValReg = getRegForValue(Val);
690 if (ValReg == 0)
691 return false;
692
693 return X86FastEmitStore(VT, ValReg, AM, MMO, Aligned);
694}
695
696/// X86FastEmitExtend - Emit a machine instruction to extend a value Src of
697/// type SrcVT to type DstVT using the specified extension opcode Opc (e.g.
698/// ISD::SIGN_EXTEND).
699bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT,
700 unsigned Src, EVT SrcVT,
701 unsigned &ResultReg) {
702 unsigned RR = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Opc, Src);
703 if (RR == 0)
704 return false;
705
706 ResultReg = RR;
707 return true;
708}
709
710bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
711 // Handle constant address.
712 if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
713 // Can't handle alternate code models yet.
714 if (TM.getCodeModel() != CodeModel::Small &&
715 TM.getCodeModel() != CodeModel::Medium)
716 return false;
717
718 // Can't handle large objects yet.
719 if (TM.isLargeGlobalValue(GV))
720 return false;
721
722 // Can't handle TLS yet.
723 if (GV->isThreadLocal())
724 return false;
725
726 // Can't handle !absolute_symbol references yet.
727 if (GV->isAbsoluteSymbolRef())
728 return false;
729
730 // RIP-relative addresses can't have additional register operands, so if
731 // we've already folded stuff into the addressing mode, just force the
732 // global value into its own register, which we can use as the basereg.
733 if (!Subtarget->isPICStyleRIPRel() ||
734 (AM.Base.Reg == 0 && AM.IndexReg == 0)) {
735 // Okay, we've committed to selecting this global. Set up the address.
736 AM.GV = GV;
737
738 // Allow the subtarget to classify the global.
739 unsigned char GVFlags = Subtarget->classifyGlobalReference(GV);
740
741 // If this reference is relative to the pic base, set it now.
742 if (isGlobalRelativeToPICBase(GVFlags)) {
743 // FIXME: How do we know Base.Reg is free??
744 AM.Base.Reg = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
745 }
746
747 // Unless the ABI requires an extra load, return a direct reference to
748 // the global.
749 if (!isGlobalStubReference(GVFlags)) {
750 if (Subtarget->isPICStyleRIPRel()) {
751 // Use rip-relative addressing if we can. Above we verified that the
752 // base and index registers are unused.
753 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
754 AM.Base.Reg = X86::RIP;
755 }
756 AM.GVOpFlags = GVFlags;
757 return true;
758 }
759
760 // Ok, we need to do a load from a stub. If we've already loaded from
761 // this stub, reuse the loaded pointer, otherwise emit the load now.
762 DenseMap<const Value *, Register>::iterator I = LocalValueMap.find(V);
763 Register LoadReg;
764 if (I != LocalValueMap.end() && I->second) {
765 LoadReg = I->second;
766 } else {
767 // Issue load from stub.
768 unsigned Opc = 0;
769 const TargetRegisterClass *RC = nullptr;
770 X86AddressMode StubAM;
771 StubAM.Base.Reg = AM.Base.Reg;
772 StubAM.GV = GV;
773 StubAM.GVOpFlags = GVFlags;
774
775 // Prepare for inserting code in the local-value area.
776 SavePoint SaveInsertPt = enterLocalValueArea();
777
778 if (TLI.getPointerTy(DL) == MVT::i64) {
779 Opc = X86::MOV64rm;
780 RC = &X86::GR64RegClass;
781 } else {
782 Opc = X86::MOV32rm;
783 RC = &X86::GR32RegClass;
784 }
785
786 if (Subtarget->isPICStyleRIPRel() || GVFlags == X86II::MO_GOTPCREL ||
788 StubAM.Base.Reg = X86::RIP;
789
790 LoadReg = createResultReg(RC);
791 MachineInstrBuilder LoadMI =
792 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), LoadReg);
793 addFullAddress(LoadMI, StubAM);
794
795 // Ok, back to normal mode.
796 leaveLocalValueArea(SaveInsertPt);
797
798 // Prevent loading GV stub multiple times in same MBB.
799 LocalValueMap[V] = LoadReg;
800 }
801
802 // Now construct the final address. Note that the Disp, Scale,
803 // and Index values may already be set here.
804 AM.Base.Reg = LoadReg;
805 AM.GV = nullptr;
806 return true;
807 }
808 }
809
810 // If all else fails, try to materialize the value in a register.
811 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
812 if (AM.Base.Reg == 0) {
813 AM.Base.Reg = getRegForValue(V);
814 return AM.Base.Reg != 0;
815 }
816 if (AM.IndexReg == 0) {
817 assert(AM.Scale == 1 && "Scale with no index!");
818 AM.IndexReg = getRegForValue(V);
819 return AM.IndexReg != 0;
820 }
821 }
822
823 return false;
824}
825
826/// X86SelectAddress - Attempt to fill in an address from the given value.
827///
828bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
830redo_gep:
831 const User *U = nullptr;
832 unsigned Opcode = Instruction::UserOp1;
833 if (const Instruction *I = dyn_cast<Instruction>(V)) {
834 // Don't walk into other basic blocks; it's possible we haven't
835 // visited them yet, so the instructions may not yet be assigned
836 // virtual registers.
837 if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(V)) ||
838 FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) {
839 Opcode = I->getOpcode();
840 U = I;
841 }
842 } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {
843 Opcode = C->getOpcode();
844 U = C;
845 }
846
847 if (PointerType *Ty = dyn_cast<PointerType>(V->getType()))
848 if (Ty->getAddressSpace() > 255)
849 // Fast instruction selection doesn't support the special
850 // address spaces.
851 return false;
852
853 switch (Opcode) {
854 default: break;
855 case Instruction::BitCast:
856 // Look past bitcasts.
857 return X86SelectAddress(U->getOperand(0), AM);
858
859 case Instruction::IntToPtr:
860 // Look past no-op inttoptrs.
861 if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
862 TLI.getPointerTy(DL))
863 return X86SelectAddress(U->getOperand(0), AM);
864 break;
865
866 case Instruction::PtrToInt:
867 // Look past no-op ptrtoints.
868 if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
869 return X86SelectAddress(U->getOperand(0), AM);
870 break;
871
872 case Instruction::Alloca: {
873 // Do static allocas.
874 const AllocaInst *A = cast<AllocaInst>(V);
876 FuncInfo.StaticAllocaMap.find(A);
877 if (SI != FuncInfo.StaticAllocaMap.end()) {
879 AM.Base.FrameIndex = SI->second;
880 return true;
881 }
882 break;
883 }
884
885 case Instruction::Add: {
886 // Adds of constants are common and easy enough.
887 if (const ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
888 uint64_t Disp = (int32_t)AM.Disp + (uint64_t)CI->getSExtValue();
889 // They have to fit in the 32-bit signed displacement field though.
890 if (isInt<32>(Disp)) {
891 AM.Disp = (uint32_t)Disp;
892 return X86SelectAddress(U->getOperand(0), AM);
893 }
894 }
895 break;
896 }
897
898 case Instruction::GetElementPtr: {
899 X86AddressMode SavedAM = AM;
900
901 // Pattern-match simple GEPs.
902 uint64_t Disp = (int32_t)AM.Disp;
903 unsigned IndexReg = AM.IndexReg;
904 unsigned Scale = AM.Scale;
906 // Iterate through the indices, folding what we can. Constants can be
907 // folded, and one dynamic index can be handled, if the scale is supported.
908 for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end();
909 i != e; ++i, ++GTI) {
910 const Value *Op = *i;
911 if (StructType *STy = GTI.getStructTypeOrNull()) {
912 const StructLayout *SL = DL.getStructLayout(STy);
913 Disp += SL->getElementOffset(cast<ConstantInt>(Op)->getZExtValue());
914 continue;
915 }
916
917 // A array/variable index is always of the form i*S where S is the
918 // constant scale size. See if we can push the scale into immediates.
919 uint64_t S = GTI.getSequentialElementStride(DL);
920 for (;;) {
921 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
922 // Constant-offset addressing.
923 Disp += CI->getSExtValue() * S;
924 break;
925 }
926 if (canFoldAddIntoGEP(U, Op)) {
927 // A compatible add with a constant operand. Fold the constant.
928 ConstantInt *CI =
929 cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
930 Disp += CI->getSExtValue() * S;
931 // Iterate on the other operand.
932 Op = cast<AddOperator>(Op)->getOperand(0);
933 continue;
934 }
935 if (IndexReg == 0 &&
936 (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&
937 (S == 1 || S == 2 || S == 4 || S == 8)) {
938 // Scaled-index addressing.
939 Scale = S;
940 IndexReg = getRegForGEPIndex(Op);
941 if (IndexReg == 0)
942 return false;
943 break;
944 }
945 // Unsupported.
946 goto unsupported_gep;
947 }
948 }
949
950 // Check for displacement overflow.
951 if (!isInt<32>(Disp))
952 break;
953
954 AM.IndexReg = IndexReg;
955 AM.Scale = Scale;
956 AM.Disp = (uint32_t)Disp;
957 GEPs.push_back(V);
958
959 if (const GetElementPtrInst *GEP =
960 dyn_cast<GetElementPtrInst>(U->getOperand(0))) {
961 // Ok, the GEP indices were covered by constant-offset and scaled-index
962 // addressing. Update the address state and move on to examining the base.
963 V = GEP;
964 goto redo_gep;
965 } else if (X86SelectAddress(U->getOperand(0), AM)) {
966 return true;
967 }
968
969 // If we couldn't merge the gep value into this addr mode, revert back to
970 // our address and just match the value instead of completely failing.
971 AM = SavedAM;
972
973 for (const Value *I : reverse(GEPs))
974 if (handleConstantAddresses(I, AM))
975 return true;
976
977 return false;
978 unsupported_gep:
979 // Ok, the GEP indices weren't all covered.
980 break;
981 }
982 }
983
984 return handleConstantAddresses(V, AM);
985}
986
987/// X86SelectCallAddress - Attempt to fill in an address from the given value.
988///
989bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
990 const User *U = nullptr;
991 unsigned Opcode = Instruction::UserOp1;
992 const Instruction *I = dyn_cast<Instruction>(V);
993 // Record if the value is defined in the same basic block.
994 //
995 // This information is crucial to know whether or not folding an
996 // operand is valid.
997 // Indeed, FastISel generates or reuses a virtual register for all
998 // operands of all instructions it selects. Obviously, the definition and
999 // its uses must use the same virtual register otherwise the produced
1000 // code is incorrect.
1001 // Before instruction selection, FunctionLoweringInfo::set sets the virtual
1002 // registers for values that are alive across basic blocks. This ensures
1003 // that the values are consistently set between across basic block, even
1004 // if different instruction selection mechanisms are used (e.g., a mix of
1005 // SDISel and FastISel).
1006 // For values local to a basic block, the instruction selection process
1007 // generates these virtual registers with whatever method is appropriate
1008 // for its needs. In particular, FastISel and SDISel do not share the way
1009 // local virtual registers are set.
1010 // Therefore, this is impossible (or at least unsafe) to share values
1011 // between basic blocks unless they use the same instruction selection
1012 // method, which is not guarantee for X86.
1013 // Moreover, things like hasOneUse could not be used accurately, if we
1014 // allow to reference values across basic blocks whereas they are not
1015 // alive across basic blocks initially.
1016 bool InMBB = true;
1017 if (I) {
1018 Opcode = I->getOpcode();
1019 U = I;
1020 InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock();
1021 } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {
1022 Opcode = C->getOpcode();
1023 U = C;
1024 }
1025
1026 switch (Opcode) {
1027 default: break;
1028 case Instruction::BitCast:
1029 // Look past bitcasts if its operand is in the same BB.
1030 if (InMBB)
1031 return X86SelectCallAddress(U->getOperand(0), AM);
1032 break;
1033
1034 case Instruction::IntToPtr:
1035 // Look past no-op inttoptrs if its operand is in the same BB.
1036 if (InMBB &&
1037 TLI.getValueType(DL, U->getOperand(0)->getType()) ==
1038 TLI.getPointerTy(DL))
1039 return X86SelectCallAddress(U->getOperand(0), AM);
1040 break;
1041
1042 case Instruction::PtrToInt:
1043 // Look past no-op ptrtoints if its operand is in the same BB.
1044 if (InMBB && TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
1045 return X86SelectCallAddress(U->getOperand(0), AM);
1046 break;
1047 }
1048
1049 // Handle constant address.
1050 if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
1051 // Can't handle alternate code models yet.
1052 if (TM.getCodeModel() != CodeModel::Small &&
1053 TM.getCodeModel() != CodeModel::Medium)
1054 return false;
1055
1056 // RIP-relative addresses can't have additional register operands.
1057 if (Subtarget->isPICStyleRIPRel() &&
1058 (AM.Base.Reg != 0 || AM.IndexReg != 0))
1059 return false;
1060
1061 // Can't handle TLS.
1062 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
1063 if (GVar->isThreadLocal())
1064 return false;
1065
1066 // Okay, we've committed to selecting this global. Set up the basic address.
1067 AM.GV = GV;
1068
1069 // Return a direct reference to the global. Fastisel can handle calls to
1070 // functions that require loads, such as dllimport and nonlazybind
1071 // functions.
1072 if (Subtarget->isPICStyleRIPRel()) {
1073 // Use rip-relative addressing if we can. Above we verified that the
1074 // base and index registers are unused.
1075 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
1076 AM.Base.Reg = X86::RIP;
1077 } else {
1078 AM.GVOpFlags = Subtarget->classifyLocalReference(nullptr);
1079 }
1080
1081 return true;
1082 }
1083
1084 // If all else fails, try to materialize the value in a register.
1085 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
1086 auto GetCallRegForValue = [this](const Value *V) {
1087 Register Reg = getRegForValue(V);
1088
1089 // In 64-bit mode, we need a 64-bit register even if pointers are 32 bits.
1090 if (Reg && Subtarget->isTarget64BitILP32()) {
1091 Register CopyReg = createResultReg(&X86::GR32RegClass);
1092 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV32rr),
1093 CopyReg)
1094 .addReg(Reg);
1095
1096 Register ExtReg = createResultReg(&X86::GR64RegClass);
1097 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1098 TII.get(TargetOpcode::SUBREG_TO_REG), ExtReg)
1099 .addImm(0)
1100 .addReg(CopyReg)
1101 .addImm(X86::sub_32bit);
1102 Reg = ExtReg;
1103 }
1104
1105 return Reg;
1106 };
1107
1108 if (AM.Base.Reg == 0) {
1109 AM.Base.Reg = GetCallRegForValue(V);
1110 return AM.Base.Reg != 0;
1111 }
1112 if (AM.IndexReg == 0) {
1113 assert(AM.Scale == 1 && "Scale with no index!");
1114 AM.IndexReg = GetCallRegForValue(V);
1115 return AM.IndexReg != 0;
1116 }
1117 }
1118
1119 return false;
1120}
1121
1122
1123/// X86SelectStore - Select and emit code to implement store instructions.
1124bool X86FastISel::X86SelectStore(const Instruction *I) {
1125 // Atomic stores need special handling.
1126 const StoreInst *S = cast<StoreInst>(I);
1127
1128 if (S->isAtomic())
1129 return false;
1130
1131 const Value *PtrV = I->getOperand(1);
1132 if (TLI.supportSwiftError()) {
1133 // Swifterror values can come from either a function parameter with
1134 // swifterror attribute or an alloca with swifterror attribute.
1135 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
1136 if (Arg->hasSwiftErrorAttr())
1137 return false;
1138 }
1139
1140 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
1141 if (Alloca->isSwiftError())
1142 return false;
1143 }
1144 }
1145
1146 const Value *Val = S->getValueOperand();
1147 const Value *Ptr = S->getPointerOperand();
1148
1149 MVT VT;
1150 if (!isTypeLegal(Val->getType(), VT, /*AllowI1=*/true))
1151 return false;
1152
1153 Align Alignment = S->getAlign();
1154 Align ABIAlignment = DL.getABITypeAlign(Val->getType());
1155 bool Aligned = Alignment >= ABIAlignment;
1156
1157 X86AddressMode AM;
1158 if (!X86SelectAddress(Ptr, AM))
1159 return false;
1160
1161 return X86FastEmitStore(VT, Val, AM, createMachineMemOperandFor(I), Aligned);
1162}
1163
1164/// X86SelectRet - Select and emit code to implement ret instructions.
1165bool X86FastISel::X86SelectRet(const Instruction *I) {
1166 const ReturnInst *Ret = cast<ReturnInst>(I);
1167 const Function &F = *I->getParent()->getParent();
1168 const X86MachineFunctionInfo *X86MFInfo =
1169 FuncInfo.MF->getInfo<X86MachineFunctionInfo>();
1170
1171 if (!FuncInfo.CanLowerReturn)
1172 return false;
1173
1174 if (TLI.supportSwiftError() &&
1175 F.getAttributes().hasAttrSomewhere(Attribute::SwiftError))
1176 return false;
1177
1178 if (TLI.supportSplitCSR(FuncInfo.MF))
1179 return false;
1180
1181 CallingConv::ID CC = F.getCallingConv();
1182 if (CC != CallingConv::C &&
1183 CC != CallingConv::Fast &&
1184 CC != CallingConv::Tail &&
1191 return false;
1192
1193 // Don't handle popping bytes if they don't fit the ret's immediate.
1194 if (!isUInt<16>(X86MFInfo->getBytesToPopOnReturn()))
1195 return false;
1196
1197 // fastcc with -tailcallopt is intended to provide a guaranteed
1198 // tail call optimization. Fastisel doesn't know how to do that.
1199 if ((CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt) ||
1201 return false;
1202
1203 // Let SDISel handle vararg functions.
1204 if (F.isVarArg())
1205 return false;
1206
1207 // Build a list of return value registers.
1209
1210 if (Ret->getNumOperands() > 0) {
1212 GetReturnInfo(CC, F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
1213
1214 // Analyze operands of the call, assigning locations to each operand.
1216 CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext());
1217 CCInfo.AnalyzeReturn(Outs, RetCC_X86);
1218
1219 const Value *RV = Ret->getOperand(0);
1220 Register Reg = getRegForValue(RV);
1221 if (Reg == 0)
1222 return false;
1223
1224 // Only handle a single return value for now.
1225 if (ValLocs.size() != 1)
1226 return false;
1227
1228 CCValAssign &VA = ValLocs[0];
1229
1230 // Don't bother handling odd stuff for now.
1231 if (VA.getLocInfo() != CCValAssign::Full)
1232 return false;
1233 // Only handle register returns for now.
1234 if (!VA.isRegLoc())
1235 return false;
1236
1237 // The calling-convention tables for x87 returns don't tell
1238 // the whole story.
1239 if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
1240 return false;
1241
1242 unsigned SrcReg = Reg + VA.getValNo();
1243 EVT SrcVT = TLI.getValueType(DL, RV->getType());
1244 EVT DstVT = VA.getValVT();
1245 // Special handling for extended integers.
1246 if (SrcVT != DstVT) {
1247 if (SrcVT != MVT::i1 && SrcVT != MVT::i8 && SrcVT != MVT::i16)
1248 return false;
1249
1250 if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
1251 return false;
1252
1253 if (SrcVT == MVT::i1) {
1254 if (Outs[0].Flags.isSExt())
1255 return false;
1256 SrcReg = fastEmitZExtFromI1(MVT::i8, SrcReg);
1257 SrcVT = MVT::i8;
1258 }
1259 if (SrcVT != DstVT) {
1260 unsigned Op =
1261 Outs[0].Flags.isZExt() ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
1262 SrcReg =
1263 fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Op, SrcReg);
1264 }
1265 }
1266
1267 // Make the copy.
1268 Register DstReg = VA.getLocReg();
1269 const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
1270 // Avoid a cross-class copy. This is very unlikely.
1271 if (!SrcRC->contains(DstReg))
1272 return false;
1273 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1274 TII.get(TargetOpcode::COPY), DstReg).addReg(SrcReg);
1275
1276 // Add register to return instruction.
1277 RetRegs.push_back(VA.getLocReg());
1278 }
1279
1280 // Swift calling convention does not require we copy the sret argument
1281 // into %rax/%eax for the return, and SRetReturnReg is not set for Swift.
1282
1283 // All x86 ABIs require that for returning structs by value we copy
1284 // the sret argument into %rax/%eax (depending on ABI) for the return.
1285 // We saved the argument into a virtual register in the entry block,
1286 // so now we copy the value out and into %rax/%eax.
1287 if (F.hasStructRetAttr() && CC != CallingConv::Swift &&
1289 Register Reg = X86MFInfo->getSRetReturnReg();
1290 assert(Reg &&
1291 "SRetReturnReg should have been set in LowerFormalArguments()!");
1292 unsigned RetReg = Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX;
1293 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1294 TII.get(TargetOpcode::COPY), RetReg).addReg(Reg);
1295 RetRegs.push_back(RetReg);
1296 }
1297
1298 // Now emit the RET.
1300 if (X86MFInfo->getBytesToPopOnReturn()) {
1301 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1302 TII.get(Subtarget->is64Bit() ? X86::RETI64 : X86::RETI32))
1303 .addImm(X86MFInfo->getBytesToPopOnReturn());
1304 } else {
1305 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1306 TII.get(Subtarget->is64Bit() ? X86::RET64 : X86::RET32));
1307 }
1308 for (unsigned Reg : RetRegs)
1309 MIB.addReg(Reg, RegState::Implicit);
1310 return true;
1311}
1312
1313/// X86SelectLoad - Select and emit code to implement load instructions.
1314///
1315bool X86FastISel::X86SelectLoad(const Instruction *I) {
1316 const LoadInst *LI = cast<LoadInst>(I);
1317
1318 // Atomic loads need special handling.
1319 if (LI->isAtomic())
1320 return false;
1321
1322 const Value *SV = I->getOperand(0);
1323 if (TLI.supportSwiftError()) {
1324 // Swifterror values can come from either a function parameter with
1325 // swifterror attribute or an alloca with swifterror attribute.
1326 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
1327 if (Arg->hasSwiftErrorAttr())
1328 return false;
1329 }
1330
1331 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
1332 if (Alloca->isSwiftError())
1333 return false;
1334 }
1335 }
1336
1337 MVT VT;
1338 if (!isTypeLegal(LI->getType(), VT, /*AllowI1=*/true))
1339 return false;
1340
1341 const Value *Ptr = LI->getPointerOperand();
1342
1343 X86AddressMode AM;
1344 if (!X86SelectAddress(Ptr, AM))
1345 return false;
1346
1347 unsigned ResultReg = 0;
1348 if (!X86FastEmitLoad(VT, AM, createMachineMemOperandFor(LI), ResultReg,
1349 LI->getAlign().value()))
1350 return false;
1351
1352 updateValueMap(I, ResultReg);
1353 return true;
1354}
1355
1356static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget) {
1357 bool HasAVX512 = Subtarget->hasAVX512();
1358 bool HasAVX = Subtarget->hasAVX();
1359 bool HasSSE1 = Subtarget->hasSSE1();
1360 bool HasSSE2 = Subtarget->hasSSE2();
1361
1362 switch (VT.getSimpleVT().SimpleTy) {
1363 default: return 0;
1364 case MVT::i8: return X86::CMP8rr;
1365 case MVT::i16: return X86::CMP16rr;
1366 case MVT::i32: return X86::CMP32rr;
1367 case MVT::i64: return X86::CMP64rr;
1368 case MVT::f32:
1369 return HasAVX512 ? X86::VUCOMISSZrr
1370 : HasAVX ? X86::VUCOMISSrr
1371 : HasSSE1 ? X86::UCOMISSrr
1372 : 0;
1373 case MVT::f64:
1374 return HasAVX512 ? X86::VUCOMISDZrr
1375 : HasAVX ? X86::VUCOMISDrr
1376 : HasSSE2 ? X86::UCOMISDrr
1377 : 0;
1378 }
1379}
1380
1381/// If we have a comparison with RHS as the RHS of the comparison, return an
1382/// opcode that works for the compare (e.g. CMP32ri) otherwise return 0.
1383static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC) {
1384 switch (VT.getSimpleVT().SimpleTy) {
1385 // Otherwise, we can't fold the immediate into this comparison.
1386 default:
1387 return 0;
1388 case MVT::i8:
1389 return X86::CMP8ri;
1390 case MVT::i16:
1391 return X86::CMP16ri;
1392 case MVT::i32:
1393 return X86::CMP32ri;
1394 case MVT::i64:
1395 // 64-bit comparisons are only valid if the immediate fits in a 32-bit sext
1396 // field.
1397 return isInt<32>(RHSC->getSExtValue()) ? X86::CMP64ri32 : 0;
1398 }
1399}
1400
1401bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1, EVT VT,
1402 const DebugLoc &CurMIMD) {
1403 Register Op0Reg = getRegForValue(Op0);
1404 if (Op0Reg == 0) return false;
1405
1406 // Handle 'null' like i32/i64 0.
1407 if (isa<ConstantPointerNull>(Op1))
1408 Op1 = Constant::getNullValue(DL.getIntPtrType(Op0->getContext()));
1409
1410 // We have two options: compare with register or immediate. If the RHS of
1411 // the compare is an immediate that we can fold into this compare, use
1412 // CMPri, otherwise use CMPrr.
1413 if (const ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) {
1414 if (unsigned CompareImmOpc = X86ChooseCmpImmediateOpcode(VT, Op1C)) {
1415 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurMIMD, TII.get(CompareImmOpc))
1416 .addReg(Op0Reg)
1417 .addImm(Op1C->getSExtValue());
1418 return true;
1419 }
1420 }
1421
1422 unsigned CompareOpc = X86ChooseCmpOpcode(VT, Subtarget);
1423 if (CompareOpc == 0) return false;
1424
1425 Register Op1Reg = getRegForValue(Op1);
1426 if (Op1Reg == 0) return false;
1427 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurMIMD, TII.get(CompareOpc))
1428 .addReg(Op0Reg)
1429 .addReg(Op1Reg);
1430
1431 return true;
1432}
1433
1434bool X86FastISel::X86SelectCmp(const Instruction *I) {
1435 const CmpInst *CI = cast<CmpInst>(I);
1436
1437 MVT VT;
1438 if (!isTypeLegal(I->getOperand(0)->getType(), VT))
1439 return false;
1440
1441 // Below code only works for scalars.
1442 if (VT.isVector())
1443 return false;
1444
1445 // Try to optimize or fold the cmp.
1446 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1447 unsigned ResultReg = 0;
1448 switch (Predicate) {
1449 default: break;
1450 case CmpInst::FCMP_FALSE: {
1451 ResultReg = createResultReg(&X86::GR32RegClass);
1452 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV32r0),
1453 ResultReg);
1454 ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultReg, X86::sub_8bit);
1455 if (!ResultReg)
1456 return false;
1457 break;
1458 }
1459 case CmpInst::FCMP_TRUE: {
1460 ResultReg = createResultReg(&X86::GR8RegClass);
1461 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV8ri),
1462 ResultReg).addImm(1);
1463 break;
1464 }
1465 }
1466
1467 if (ResultReg) {
1468 updateValueMap(I, ResultReg);
1469 return true;
1470 }
1471
1472 const Value *LHS = CI->getOperand(0);
1473 const Value *RHS = CI->getOperand(1);
1474
1475 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
1476 // We don't have to materialize a zero constant for this case and can just use
1477 // %x again on the RHS.
1478 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1479 const auto *RHSC = dyn_cast<ConstantFP>(RHS);
1480 if (RHSC && RHSC->isNullValue())
1481 RHS = LHS;
1482 }
1483
1484 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1485 static const uint16_t SETFOpcTable[2][3] = {
1486 { X86::COND_E, X86::COND_NP, X86::AND8rr },
1487 { X86::COND_NE, X86::COND_P, X86::OR8rr }
1488 };
1489 const uint16_t *SETFOpc = nullptr;
1490 switch (Predicate) {
1491 default: break;
1492 case CmpInst::FCMP_OEQ: SETFOpc = &SETFOpcTable[0][0]; break;
1493 case CmpInst::FCMP_UNE: SETFOpc = &SETFOpcTable[1][0]; break;
1494 }
1495
1496 ResultReg = createResultReg(&X86::GR8RegClass);
1497 if (SETFOpc) {
1498 if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))
1499 return false;
1500
1501 Register FlagReg1 = createResultReg(&X86::GR8RegClass);
1502 Register FlagReg2 = createResultReg(&X86::GR8RegClass);
1503 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
1504 FlagReg1).addImm(SETFOpc[0]);
1505 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
1506 FlagReg2).addImm(SETFOpc[1]);
1507 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(SETFOpc[2]),
1508 ResultReg).addReg(FlagReg1).addReg(FlagReg2);
1509 updateValueMap(I, ResultReg);
1510 return true;
1511 }
1512
1514 bool SwapArgs;
1515 std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1516 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1517
1518 if (SwapArgs)
1519 std::swap(LHS, RHS);
1520
1521 // Emit a compare of LHS/RHS.
1522 if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))
1523 return false;
1524
1525 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
1526 ResultReg).addImm(CC);
1527 updateValueMap(I, ResultReg);
1528 return true;
1529}
1530
1531bool X86FastISel::X86SelectZExt(const Instruction *I) {
1532 EVT DstVT = TLI.getValueType(DL, I->getType());
1533 if (!TLI.isTypeLegal(DstVT))
1534 return false;
1535
1536 Register ResultReg = getRegForValue(I->getOperand(0));
1537 if (ResultReg == 0)
1538 return false;
1539
1540 // Handle zero-extension from i1 to i8, which is common.
1541 MVT SrcVT = TLI.getSimpleValueType(DL, I->getOperand(0)->getType());
1542 if (SrcVT == MVT::i1) {
1543 // Set the high bits to zero.
1544 ResultReg = fastEmitZExtFromI1(MVT::i8, ResultReg);
1545 SrcVT = MVT::i8;
1546
1547 if (ResultReg == 0)
1548 return false;
1549 }
1550
1551 if (DstVT == MVT::i64) {
1552 // Handle extension to 64-bits via sub-register shenanigans.
1553 unsigned MovInst;
1554
1555 switch (SrcVT.SimpleTy) {
1556 case MVT::i8: MovInst = X86::MOVZX32rr8; break;
1557 case MVT::i16: MovInst = X86::MOVZX32rr16; break;
1558 case MVT::i32: MovInst = X86::MOV32rr; break;
1559 default: llvm_unreachable("Unexpected zext to i64 source type");
1560 }
1561
1562 Register Result32 = createResultReg(&X86::GR32RegClass);
1563 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(MovInst), Result32)
1564 .addReg(ResultReg);
1565
1566 ResultReg = createResultReg(&X86::GR64RegClass);
1567 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::SUBREG_TO_REG),
1568 ResultReg)
1569 .addImm(0).addReg(Result32).addImm(X86::sub_32bit);
1570 } else if (DstVT == MVT::i16) {
1571 // i8->i16 doesn't exist in the autogenerated isel table. Need to zero
1572 // extend to 32-bits and then extract down to 16-bits.
1573 Register Result32 = createResultReg(&X86::GR32RegClass);
1574 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOVZX32rr8),
1575 Result32).addReg(ResultReg);
1576
1577 ResultReg = fastEmitInst_extractsubreg(MVT::i16, Result32, X86::sub_16bit);
1578 } else if (DstVT != MVT::i8) {
1579 ResultReg = fastEmit_r(MVT::i8, DstVT.getSimpleVT(), ISD::ZERO_EXTEND,
1580 ResultReg);
1581 if (ResultReg == 0)
1582 return false;
1583 }
1584
1585 updateValueMap(I, ResultReg);
1586 return true;
1587}
1588
1589bool X86FastISel::X86SelectSExt(const Instruction *I) {
1590 EVT DstVT = TLI.getValueType(DL, I->getType());
1591 if (!TLI.isTypeLegal(DstVT))
1592 return false;
1593
1594 Register ResultReg = getRegForValue(I->getOperand(0));
1595 if (ResultReg == 0)
1596 return false;
1597
1598 // Handle sign-extension from i1 to i8.
1599 MVT SrcVT = TLI.getSimpleValueType(DL, I->getOperand(0)->getType());
1600 if (SrcVT == MVT::i1) {
1601 // Set the high bits to zero.
1602 Register ZExtReg = fastEmitZExtFromI1(MVT::i8, ResultReg);
1603 if (ZExtReg == 0)
1604 return false;
1605
1606 // Negate the result to make an 8-bit sign extended value.
1607 ResultReg = createResultReg(&X86::GR8RegClass);
1608 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::NEG8r),
1609 ResultReg).addReg(ZExtReg);
1610
1611 SrcVT = MVT::i8;
1612 }
1613
1614 if (DstVT == MVT::i16) {
1615 // i8->i16 doesn't exist in the autogenerated isel table. Need to sign
1616 // extend to 32-bits and then extract down to 16-bits.
1617 Register Result32 = createResultReg(&X86::GR32RegClass);
1618 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOVSX32rr8),
1619 Result32).addReg(ResultReg);
1620
1621 ResultReg = fastEmitInst_extractsubreg(MVT::i16, Result32, X86::sub_16bit);
1622 } else if (DstVT != MVT::i8) {
1623 ResultReg = fastEmit_r(MVT::i8, DstVT.getSimpleVT(), ISD::SIGN_EXTEND,
1624 ResultReg);
1625 if (ResultReg == 0)
1626 return false;
1627 }
1628
1629 updateValueMap(I, ResultReg);
1630 return true;
1631}
1632
1633bool X86FastISel::X86SelectBranch(const Instruction *I) {
1634 // Unconditional branches are selected by tablegen-generated code.
1635 // Handle a conditional branch.
1636 const BranchInst *BI = cast<BranchInst>(I);
1637 MachineBasicBlock *TrueMBB = FuncInfo.MBBMap[BI->getSuccessor(0)];
1638 MachineBasicBlock *FalseMBB = FuncInfo.MBBMap[BI->getSuccessor(1)];
1639
1640 // Fold the common case of a conditional branch with a comparison
1641 // in the same block (values defined on other blocks may not have
1642 // initialized registers).
1644 if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) {
1645 if (CI->hasOneUse() && CI->getParent() == I->getParent()) {
1646 EVT VT = TLI.getValueType(DL, CI->getOperand(0)->getType());
1647
1648 // Try to optimize or fold the cmp.
1649 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1650 switch (Predicate) {
1651 default: break;
1652 case CmpInst::FCMP_FALSE: fastEmitBranch(FalseMBB, MIMD.getDL()); return true;
1653 case CmpInst::FCMP_TRUE: fastEmitBranch(TrueMBB, MIMD.getDL()); return true;
1654 }
1655
1656 const Value *CmpLHS = CI->getOperand(0);
1657 const Value *CmpRHS = CI->getOperand(1);
1658
1659 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x,
1660 // 0.0.
1661 // We don't have to materialize a zero constant for this case and can just
1662 // use %x again on the RHS.
1663 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1664 const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);
1665 if (CmpRHSC && CmpRHSC->isNullValue())
1666 CmpRHS = CmpLHS;
1667 }
1668
1669 // Try to take advantage of fallthrough opportunities.
1670 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
1671 std::swap(TrueMBB, FalseMBB);
1673 }
1674
1675 // FCMP_OEQ and FCMP_UNE cannot be expressed with a single flag/condition
1676 // code check. Instead two branch instructions are required to check all
1677 // the flags. First we change the predicate to a supported condition code,
1678 // which will be the first branch. Later one we will emit the second
1679 // branch.
1680 bool NeedExtraBranch = false;
1681 switch (Predicate) {
1682 default: break;
1683 case CmpInst::FCMP_OEQ:
1684 std::swap(TrueMBB, FalseMBB);
1685 [[fallthrough]];
1686 case CmpInst::FCMP_UNE:
1687 NeedExtraBranch = true;
1689 break;
1690 }
1691
1692 bool SwapArgs;
1693 std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1694 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1695
1696 if (SwapArgs)
1697 std::swap(CmpLHS, CmpRHS);
1698
1699 // Emit a compare of the LHS and RHS, setting the flags.
1700 if (!X86FastEmitCompare(CmpLHS, CmpRHS, VT, CI->getDebugLoc()))
1701 return false;
1702
1703 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1704 .addMBB(TrueMBB).addImm(CC);
1705
1706 // X86 requires a second branch to handle UNE (and OEQ, which is mapped
1707 // to UNE above).
1708 if (NeedExtraBranch) {
1709 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1710 .addMBB(TrueMBB).addImm(X86::COND_P);
1711 }
1712
1713 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1714 return true;
1715 }
1716 } else if (TruncInst *TI = dyn_cast<TruncInst>(BI->getCondition())) {
1717 // Handle things like "%cond = trunc i32 %X to i1 / br i1 %cond", which
1718 // typically happen for _Bool and C++ bools.
1719 MVT SourceVT;
1720 if (TI->hasOneUse() && TI->getParent() == I->getParent() &&
1721 isTypeLegal(TI->getOperand(0)->getType(), SourceVT)) {
1722 unsigned TestOpc = 0;
1723 switch (SourceVT.SimpleTy) {
1724 default: break;
1725 case MVT::i8: TestOpc = X86::TEST8ri; break;
1726 case MVT::i16: TestOpc = X86::TEST16ri; break;
1727 case MVT::i32: TestOpc = X86::TEST32ri; break;
1728 case MVT::i64: TestOpc = X86::TEST64ri32; break;
1729 }
1730 if (TestOpc) {
1731 Register OpReg = getRegForValue(TI->getOperand(0));
1732 if (OpReg == 0) return false;
1733
1734 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TestOpc))
1735 .addReg(OpReg).addImm(1);
1736
1737 unsigned JmpCond = X86::COND_NE;
1738 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
1739 std::swap(TrueMBB, FalseMBB);
1740 JmpCond = X86::COND_E;
1741 }
1742
1743 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1744 .addMBB(TrueMBB).addImm(JmpCond);
1745
1746 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1747 return true;
1748 }
1749 }
1750 } else if (foldX86XALUIntrinsic(CC, BI, BI->getCondition())) {
1751 // Fake request the condition, otherwise the intrinsic might be completely
1752 // optimized away.
1753 Register TmpReg = getRegForValue(BI->getCondition());
1754 if (TmpReg == 0)
1755 return false;
1756
1757 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1758 .addMBB(TrueMBB).addImm(CC);
1759 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1760 return true;
1761 }
1762
1763 // Otherwise do a clumsy setcc and re-test it.
1764 // Note that i1 essentially gets ANY_EXTEND'ed to i8 where it isn't used
1765 // in an explicit cast, so make sure to handle that correctly.
1766 Register OpReg = getRegForValue(BI->getCondition());
1767 if (OpReg == 0) return false;
1768
1769 // In case OpReg is a K register, COPY to a GPR
1770 if (MRI.getRegClass(OpReg) == &X86::VK1RegClass) {
1771 unsigned KOpReg = OpReg;
1772 OpReg = createResultReg(&X86::GR32RegClass);
1773 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1774 TII.get(TargetOpcode::COPY), OpReg)
1775 .addReg(KOpReg);
1776 OpReg = fastEmitInst_extractsubreg(MVT::i8, OpReg, X86::sub_8bit);
1777 }
1778 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::TEST8ri))
1779 .addReg(OpReg)
1780 .addImm(1);
1781 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1782 .addMBB(TrueMBB).addImm(X86::COND_NE);
1783 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1784 return true;
1785}
1786
1787bool X86FastISel::X86SelectShift(const Instruction *I) {
1788 unsigned CReg = 0, OpReg = 0;
1789 const TargetRegisterClass *RC = nullptr;
1790 if (I->getType()->isIntegerTy(8)) {
1791 CReg = X86::CL;
1792 RC = &X86::GR8RegClass;
1793 switch (I->getOpcode()) {
1794 case Instruction::LShr: OpReg = X86::SHR8rCL; break;
1795 case Instruction::AShr: OpReg = X86::SAR8rCL; break;
1796 case Instruction::Shl: OpReg = X86::SHL8rCL; break;
1797 default: return false;
1798 }
1799 } else if (I->getType()->isIntegerTy(16)) {
1800 CReg = X86::CX;
1801 RC = &X86::GR16RegClass;
1802 switch (I->getOpcode()) {
1803 default: llvm_unreachable("Unexpected shift opcode");
1804 case Instruction::LShr: OpReg = X86::SHR16rCL; break;
1805 case Instruction::AShr: OpReg = X86::SAR16rCL; break;
1806 case Instruction::Shl: OpReg = X86::SHL16rCL; break;
1807 }
1808 } else if (I->getType()->isIntegerTy(32)) {
1809 CReg = X86::ECX;
1810 RC = &X86::GR32RegClass;
1811 switch (I->getOpcode()) {
1812 default: llvm_unreachable("Unexpected shift opcode");
1813 case Instruction::LShr: OpReg = X86::SHR32rCL; break;
1814 case Instruction::AShr: OpReg = X86::SAR32rCL; break;
1815 case Instruction::Shl: OpReg = X86::SHL32rCL; break;
1816 }
1817 } else if (I->getType()->isIntegerTy(64)) {
1818 CReg = X86::RCX;
1819 RC = &X86::GR64RegClass;
1820 switch (I->getOpcode()) {
1821 default: llvm_unreachable("Unexpected shift opcode");
1822 case Instruction::LShr: OpReg = X86::SHR64rCL; break;
1823 case Instruction::AShr: OpReg = X86::SAR64rCL; break;
1824 case Instruction::Shl: OpReg = X86::SHL64rCL; break;
1825 }
1826 } else {
1827 return false;
1828 }
1829
1830 MVT VT;
1831 if (!isTypeLegal(I->getType(), VT))
1832 return false;
1833
1834 Register Op0Reg = getRegForValue(I->getOperand(0));
1835 if (Op0Reg == 0) return false;
1836
1837 Register Op1Reg = getRegForValue(I->getOperand(1));
1838 if (Op1Reg == 0) return false;
1839 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
1840 CReg).addReg(Op1Reg);
1841
1842 // The shift instruction uses X86::CL. If we defined a super-register
1843 // of X86::CL, emit a subreg KILL to precisely describe what we're doing here.
1844 if (CReg != X86::CL)
1845 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1846 TII.get(TargetOpcode::KILL), X86::CL)
1847 .addReg(CReg, RegState::Kill);
1848
1849 Register ResultReg = createResultReg(RC);
1850 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(OpReg), ResultReg)
1851 .addReg(Op0Reg);
1852 updateValueMap(I, ResultReg);
1853 return true;
1854}
1855
1856bool X86FastISel::X86SelectDivRem(const Instruction *I) {
1857 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1858 const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem
1859 const static bool S = true; // IsSigned
1860 const static bool U = false; // !IsSigned
1861 const static unsigned Copy = TargetOpcode::COPY;
1862 // For the X86 DIV/IDIV instruction, in most cases the dividend
1863 // (numerator) must be in a specific register pair highreg:lowreg,
1864 // producing the quotient in lowreg and the remainder in highreg.
1865 // For most data types, to set up the instruction, the dividend is
1866 // copied into lowreg, and lowreg is sign-extended or zero-extended
1867 // into highreg. The exception is i8, where the dividend is defined
1868 // as a single register rather than a register pair, and we
1869 // therefore directly sign-extend or zero-extend the dividend into
1870 // lowreg, instead of copying, and ignore the highreg.
1871 const static struct DivRemEntry {
1872 // The following portion depends only on the data type.
1873 const TargetRegisterClass *RC;
1874 unsigned LowInReg; // low part of the register pair
1875 unsigned HighInReg; // high part of the register pair
1876 // The following portion depends on both the data type and the operation.
1877 struct DivRemResult {
1878 unsigned OpDivRem; // The specific DIV/IDIV opcode to use.
1879 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1880 // highreg, or copying a zero into highreg.
1881 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1882 // zero/sign-extending into lowreg for i8.
1883 unsigned DivRemResultReg; // Register containing the desired result.
1884 bool IsOpSigned; // Whether to use signed or unsigned form.
1885 } ResultTable[NumOps];
1886 } OpTable[NumTypes] = {
1887 { &X86::GR8RegClass, X86::AX, 0, {
1888 { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S }, // SDiv
1889 { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S }, // SRem
1890 { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U }, // UDiv
1891 { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U }, // URem
1892 }
1893 }, // i8
1894 { &X86::GR16RegClass, X86::AX, X86::DX, {
1895 { X86::IDIV16r, X86::CWD, Copy, X86::AX, S }, // SDiv
1896 { X86::IDIV16r, X86::CWD, Copy, X86::DX, S }, // SRem
1897 { X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U }, // UDiv
1898 { X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U }, // URem
1899 }
1900 }, // i16
1901 { &X86::GR32RegClass, X86::EAX, X86::EDX, {
1902 { X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S }, // SDiv
1903 { X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S }, // SRem
1904 { X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U }, // UDiv
1905 { X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U }, // URem
1906 }
1907 }, // i32
1908 { &X86::GR64RegClass, X86::RAX, X86::RDX, {
1909 { X86::IDIV64r, X86::CQO, Copy, X86::RAX, S }, // SDiv
1910 { X86::IDIV64r, X86::CQO, Copy, X86::RDX, S }, // SRem
1911 { X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U }, // UDiv
1912 { X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U }, // URem
1913 }
1914 }, // i64
1915 };
1916
1917 MVT VT;
1918 if (!isTypeLegal(I->getType(), VT))
1919 return false;
1920
1921 unsigned TypeIndex, OpIndex;
1922 switch (VT.SimpleTy) {
1923 default: return false;
1924 case MVT::i8: TypeIndex = 0; break;
1925 case MVT::i16: TypeIndex = 1; break;
1926 case MVT::i32: TypeIndex = 2; break;
1927 case MVT::i64: TypeIndex = 3;
1928 if (!Subtarget->is64Bit())
1929 return false;
1930 break;
1931 }
1932
1933 switch (I->getOpcode()) {
1934 default: llvm_unreachable("Unexpected div/rem opcode");
1935 case Instruction::SDiv: OpIndex = 0; break;
1936 case Instruction::SRem: OpIndex = 1; break;
1937 case Instruction::UDiv: OpIndex = 2; break;
1938 case Instruction::URem: OpIndex = 3; break;
1939 }
1940
1941 const DivRemEntry &TypeEntry = OpTable[TypeIndex];
1942 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1943 Register Op0Reg = getRegForValue(I->getOperand(0));
1944 if (Op0Reg == 0)
1945 return false;
1946 Register Op1Reg = getRegForValue(I->getOperand(1));
1947 if (Op1Reg == 0)
1948 return false;
1949
1950 // Move op0 into low-order input register.
1951 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1952 TII.get(OpEntry.OpCopy), TypeEntry.LowInReg).addReg(Op0Reg);
1953 // Zero-extend or sign-extend into high-order input register.
1954 if (OpEntry.OpSignExtend) {
1955 if (OpEntry.IsOpSigned)
1956 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1957 TII.get(OpEntry.OpSignExtend));
1958 else {
1959 Register Zero32 = createResultReg(&X86::GR32RegClass);
1960 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1961 TII.get(X86::MOV32r0), Zero32);
1962
1963 // Copy the zero into the appropriate sub/super/identical physical
1964 // register. Unfortunately the operations needed are not uniform enough
1965 // to fit neatly into the table above.
1966 if (VT == MVT::i16) {
1967 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1968 TII.get(Copy), TypeEntry.HighInReg)
1969 .addReg(Zero32, 0, X86::sub_16bit);
1970 } else if (VT == MVT::i32) {
1971 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1972 TII.get(Copy), TypeEntry.HighInReg)
1973 .addReg(Zero32);
1974 } else if (VT == MVT::i64) {
1975 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1976 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1977 .addImm(0).addReg(Zero32).addImm(X86::sub_32bit);
1978 }
1979 }
1980 }
1981 // Generate the DIV/IDIV instruction.
1982 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1983 TII.get(OpEntry.OpDivRem)).addReg(Op1Reg);
1984 // For i8 remainder, we can't reference ah directly, as we'll end
1985 // up with bogus copies like %r9b = COPY %ah. Reference ax
1986 // instead to prevent ah references in a rex instruction.
1987 //
1988 // The current assumption of the fast register allocator is that isel
1989 // won't generate explicit references to the GR8_NOREX registers. If
1990 // the allocator and/or the backend get enhanced to be more robust in
1991 // that regard, this can be, and should be, removed.
1992 unsigned ResultReg = 0;
1993 if ((I->getOpcode() == Instruction::SRem ||
1994 I->getOpcode() == Instruction::URem) &&
1995 OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {
1996 Register SourceSuperReg = createResultReg(&X86::GR16RegClass);
1997 Register ResultSuperReg = createResultReg(&X86::GR16RegClass);
1998 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1999 TII.get(Copy), SourceSuperReg).addReg(X86::AX);
2000
2001 // Shift AX right by 8 bits instead of using AH.
2002 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SHR16ri),
2003 ResultSuperReg).addReg(SourceSuperReg).addImm(8);
2004
2005 // Now reference the 8-bit subreg of the result.
2006 ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultSuperReg,
2007 X86::sub_8bit);
2008 }
2009 // Copy the result out of the physreg if we haven't already.
2010 if (!ResultReg) {
2011 ResultReg = createResultReg(TypeEntry.RC);
2012 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Copy), ResultReg)
2013 .addReg(OpEntry.DivRemResultReg);
2014 }
2015 updateValueMap(I, ResultReg);
2016
2017 return true;
2018}
2019
2020/// Emit a conditional move instruction (if the are supported) to lower
2021/// the select.
2022bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
2023 // Check if the subtarget supports these instructions.
2024 if (!Subtarget->canUseCMOV())
2025 return false;
2026
2027 // FIXME: Add support for i8.
2028 if (RetVT < MVT::i16 || RetVT > MVT::i64)
2029 return false;
2030
2031 const Value *Cond = I->getOperand(0);
2032 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2033 bool NeedTest = true;
2035
2036 // Optimize conditions coming from a compare if both instructions are in the
2037 // same basic block (values defined in other basic blocks may not have
2038 // initialized registers).
2039 const auto *CI = dyn_cast<CmpInst>(Cond);
2040 if (CI && (CI->getParent() == I->getParent())) {
2041 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2042
2043 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
2044 static const uint16_t SETFOpcTable[2][3] = {
2045 { X86::COND_NP, X86::COND_E, X86::TEST8rr },
2046 { X86::COND_P, X86::COND_NE, X86::OR8rr }
2047 };
2048 const uint16_t *SETFOpc = nullptr;
2049 switch (Predicate) {
2050 default: break;
2051 case CmpInst::FCMP_OEQ:
2052 SETFOpc = &SETFOpcTable[0][0];
2054 break;
2055 case CmpInst::FCMP_UNE:
2056 SETFOpc = &SETFOpcTable[1][0];
2058 break;
2059 }
2060
2061 bool NeedSwap;
2062 std::tie(CC, NeedSwap) = X86::getX86ConditionCode(Predicate);
2063 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
2064
2065 const Value *CmpLHS = CI->getOperand(0);
2066 const Value *CmpRHS = CI->getOperand(1);
2067 if (NeedSwap)
2068 std::swap(CmpLHS, CmpRHS);
2069
2070 EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
2071 // Emit a compare of the LHS and RHS, setting the flags.
2072 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
2073 return false;
2074
2075 if (SETFOpc) {
2076 Register FlagReg1 = createResultReg(&X86::GR8RegClass);
2077 Register FlagReg2 = createResultReg(&X86::GR8RegClass);
2078 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
2079 FlagReg1).addImm(SETFOpc[0]);
2080 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
2081 FlagReg2).addImm(SETFOpc[1]);
2082 auto const &II = TII.get(SETFOpc[2]);
2083 if (II.getNumDefs()) {
2084 Register TmpReg = createResultReg(&X86::GR8RegClass);
2085 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, TmpReg)
2086 .addReg(FlagReg2).addReg(FlagReg1);
2087 } else {
2088 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2089 .addReg(FlagReg2).addReg(FlagReg1);
2090 }
2091 }
2092 NeedTest = false;
2093 } else if (foldX86XALUIntrinsic(CC, I, Cond)) {
2094 // Fake request the condition, otherwise the intrinsic might be completely
2095 // optimized away.
2096 Register TmpReg = getRegForValue(Cond);
2097 if (TmpReg == 0)
2098 return false;
2099
2100 NeedTest = false;
2101 }
2102
2103 if (NeedTest) {
2104 // Selects operate on i1, however, CondReg is 8 bits width and may contain
2105 // garbage. Indeed, only the less significant bit is supposed to be
2106 // accurate. If we read more than the lsb, we may see non-zero values
2107 // whereas lsb is zero. Therefore, we have to truncate Op0Reg to i1 for
2108 // the select. This is achieved by performing TEST against 1.
2109 Register CondReg = getRegForValue(Cond);
2110 if (CondReg == 0)
2111 return false;
2112
2113 // In case OpReg is a K register, COPY to a GPR
2114 if (MRI.getRegClass(CondReg) == &X86::VK1RegClass) {
2115 unsigned KCondReg = CondReg;
2116 CondReg = createResultReg(&X86::GR32RegClass);
2117 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2118 TII.get(TargetOpcode::COPY), CondReg)
2119 .addReg(KCondReg);
2120 CondReg = fastEmitInst_extractsubreg(MVT::i8, CondReg, X86::sub_8bit);
2121 }
2122 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::TEST8ri))
2123 .addReg(CondReg)
2124 .addImm(1);
2125 }
2126
2127 const Value *LHS = I->getOperand(1);
2128 const Value *RHS = I->getOperand(2);
2129
2130 Register RHSReg = getRegForValue(RHS);
2131 Register LHSReg = getRegForValue(LHS);
2132 if (!LHSReg || !RHSReg)
2133 return false;
2134
2135 const TargetRegisterInfo &TRI = *Subtarget->getRegisterInfo();
2136 unsigned Opc = X86::getCMovOpcode(TRI.getRegSizeInBits(*RC) / 8, false,
2137 Subtarget->hasNDD());
2138 Register ResultReg = fastEmitInst_rri(Opc, RC, RHSReg, LHSReg, CC);
2139 updateValueMap(I, ResultReg);
2140 return true;
2141}
2142
2143/// Emit SSE or AVX instructions to lower the select.
2144///
2145/// Try to use SSE1/SSE2 instructions to simulate a select without branches.
2146/// This lowers fp selects into a CMP/AND/ANDN/OR sequence when the necessary
2147/// SSE instructions are available. If AVX is available, try to use a VBLENDV.
2148bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {
2149 // Optimize conditions coming from a compare if both instructions are in the
2150 // same basic block (values defined in other basic blocks may not have
2151 // initialized registers).
2152 const auto *CI = dyn_cast<FCmpInst>(I->getOperand(0));
2153 if (!CI || (CI->getParent() != I->getParent()))
2154 return false;
2155
2156 if (I->getType() != CI->getOperand(0)->getType() ||
2157 !((Subtarget->hasSSE1() && RetVT == MVT::f32) ||
2158 (Subtarget->hasSSE2() && RetVT == MVT::f64)))
2159 return false;
2160
2161 const Value *CmpLHS = CI->getOperand(0);
2162 const Value *CmpRHS = CI->getOperand(1);
2163 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2164
2165 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
2166 // We don't have to materialize a zero constant for this case and can just use
2167 // %x again on the RHS.
2168 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
2169 const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);
2170 if (CmpRHSC && CmpRHSC->isNullValue())
2171 CmpRHS = CmpLHS;
2172 }
2173
2174 unsigned CC;
2175 bool NeedSwap;
2176 std::tie(CC, NeedSwap) = getX86SSEConditionCode(Predicate);
2177 if (CC > 7 && !Subtarget->hasAVX())
2178 return false;
2179
2180 if (NeedSwap)
2181 std::swap(CmpLHS, CmpRHS);
2182
2183 const Value *LHS = I->getOperand(1);
2184 const Value *RHS = I->getOperand(2);
2185
2186 Register LHSReg = getRegForValue(LHS);
2187 Register RHSReg = getRegForValue(RHS);
2188 Register CmpLHSReg = getRegForValue(CmpLHS);
2189 Register CmpRHSReg = getRegForValue(CmpRHS);
2190 if (!LHSReg || !RHSReg || !CmpLHSReg || !CmpRHSReg)
2191 return false;
2192
2193 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2194 unsigned ResultReg;
2195
2196 if (Subtarget->hasAVX512()) {
2197 // If we have AVX512 we can use a mask compare and masked movss/sd.
2198 const TargetRegisterClass *VR128X = &X86::VR128XRegClass;
2199 const TargetRegisterClass *VK1 = &X86::VK1RegClass;
2200
2201 unsigned CmpOpcode =
2202 (RetVT == MVT::f32) ? X86::VCMPSSZrri : X86::VCMPSDZrri;
2203 Register CmpReg = fastEmitInst_rri(CmpOpcode, VK1, CmpLHSReg, CmpRHSReg,
2204 CC);
2205
2206 // Need an IMPLICIT_DEF for the input that is used to generate the upper
2207 // bits of the result register since its not based on any of the inputs.
2208 Register ImplicitDefReg = createResultReg(VR128X);
2209 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2210 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2211
2212 // Place RHSReg is the passthru of the masked movss/sd operation and put
2213 // LHS in the input. The mask input comes from the compare.
2214 unsigned MovOpcode =
2215 (RetVT == MVT::f32) ? X86::VMOVSSZrrk : X86::VMOVSDZrrk;
2216 unsigned MovReg = fastEmitInst_rrrr(MovOpcode, VR128X, RHSReg, CmpReg,
2217 ImplicitDefReg, LHSReg);
2218
2219 ResultReg = createResultReg(RC);
2220 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2221 TII.get(TargetOpcode::COPY), ResultReg).addReg(MovReg);
2222
2223 } else if (Subtarget->hasAVX()) {
2224 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
2225
2226 // If we have AVX, create 1 blendv instead of 3 logic instructions.
2227 // Blendv was introduced with SSE 4.1, but the 2 register form implicitly
2228 // uses XMM0 as the selection register. That may need just as many
2229 // instructions as the AND/ANDN/OR sequence due to register moves, so
2230 // don't bother.
2231 unsigned CmpOpcode =
2232 (RetVT == MVT::f32) ? X86::VCMPSSrri : X86::VCMPSDrri;
2233 unsigned BlendOpcode =
2234 (RetVT == MVT::f32) ? X86::VBLENDVPSrrr : X86::VBLENDVPDrrr;
2235
2236 Register CmpReg = fastEmitInst_rri(CmpOpcode, RC, CmpLHSReg, CmpRHSReg,
2237 CC);
2238 Register VBlendReg = fastEmitInst_rrr(BlendOpcode, VR128, RHSReg, LHSReg,
2239 CmpReg);
2240 ResultReg = createResultReg(RC);
2241 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2242 TII.get(TargetOpcode::COPY), ResultReg).addReg(VBlendReg);
2243 } else {
2244 // Choose the SSE instruction sequence based on data type (float or double).
2245 static const uint16_t OpcTable[2][4] = {
2246 { X86::CMPSSrri, X86::ANDPSrr, X86::ANDNPSrr, X86::ORPSrr },
2247 { X86::CMPSDrri, X86::ANDPDrr, X86::ANDNPDrr, X86::ORPDrr }
2248 };
2249
2250 const uint16_t *Opc = nullptr;
2251 switch (RetVT.SimpleTy) {
2252 default: return false;
2253 case MVT::f32: Opc = &OpcTable[0][0]; break;
2254 case MVT::f64: Opc = &OpcTable[1][0]; break;
2255 }
2256
2257 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
2258 Register CmpReg = fastEmitInst_rri(Opc[0], RC, CmpLHSReg, CmpRHSReg, CC);
2259 Register AndReg = fastEmitInst_rr(Opc[1], VR128, CmpReg, LHSReg);
2260 Register AndNReg = fastEmitInst_rr(Opc[2], VR128, CmpReg, RHSReg);
2261 Register OrReg = fastEmitInst_rr(Opc[3], VR128, AndNReg, AndReg);
2262 ResultReg = createResultReg(RC);
2263 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2264 TII.get(TargetOpcode::COPY), ResultReg).addReg(OrReg);
2265 }
2266 updateValueMap(I, ResultReg);
2267 return true;
2268}
2269
2270bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {
2271 // These are pseudo CMOV instructions and will be later expanded into control-
2272 // flow.
2273 unsigned Opc;
2274 switch (RetVT.SimpleTy) {
2275 default: return false;
2276 case MVT::i8: Opc = X86::CMOV_GR8; break;
2277 case MVT::i16: Opc = X86::CMOV_GR16; break;
2278 case MVT::i32: Opc = X86::CMOV_GR32; break;
2279 case MVT::f16:
2280 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR16X : X86::CMOV_FR16; break;
2281 case MVT::f32:
2282 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR32X : X86::CMOV_FR32; break;
2283 case MVT::f64:
2284 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR64X : X86::CMOV_FR64; break;
2285 }
2286
2287 const Value *Cond = I->getOperand(0);
2289
2290 // Optimize conditions coming from a compare if both instructions are in the
2291 // same basic block (values defined in other basic blocks may not have
2292 // initialized registers).
2293 const auto *CI = dyn_cast<CmpInst>(Cond);
2294 if (CI && (CI->getParent() == I->getParent())) {
2295 bool NeedSwap;
2296 std::tie(CC, NeedSwap) = X86::getX86ConditionCode(CI->getPredicate());
2298 return false;
2299
2300 const Value *CmpLHS = CI->getOperand(0);
2301 const Value *CmpRHS = CI->getOperand(1);
2302
2303 if (NeedSwap)
2304 std::swap(CmpLHS, CmpRHS);
2305
2306 EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
2307 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
2308 return false;
2309 } else {
2310 Register CondReg = getRegForValue(Cond);
2311 if (CondReg == 0)
2312 return false;
2313
2314 // In case OpReg is a K register, COPY to a GPR
2315 if (MRI.getRegClass(CondReg) == &X86::VK1RegClass) {
2316 unsigned KCondReg = CondReg;
2317 CondReg = createResultReg(&X86::GR32RegClass);
2318 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2319 TII.get(TargetOpcode::COPY), CondReg)
2320 .addReg(KCondReg);
2321 CondReg = fastEmitInst_extractsubreg(MVT::i8, CondReg, X86::sub_8bit);
2322 }
2323 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::TEST8ri))
2324 .addReg(CondReg)
2325 .addImm(1);
2326 }
2327
2328 const Value *LHS = I->getOperand(1);
2329 const Value *RHS = I->getOperand(2);
2330
2331 Register LHSReg = getRegForValue(LHS);
2332 Register RHSReg = getRegForValue(RHS);
2333 if (!LHSReg || !RHSReg)
2334 return false;
2335
2336 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2337
2338 Register ResultReg =
2339 fastEmitInst_rri(Opc, RC, RHSReg, LHSReg, CC);
2340 updateValueMap(I, ResultReg);
2341 return true;
2342}
2343
2344bool X86FastISel::X86SelectSelect(const Instruction *I) {
2345 MVT RetVT;
2346 if (!isTypeLegal(I->getType(), RetVT))
2347 return false;
2348
2349 // Check if we can fold the select.
2350 if (const auto *CI = dyn_cast<CmpInst>(I->getOperand(0))) {
2351 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2352 const Value *Opnd = nullptr;
2353 switch (Predicate) {
2354 default: break;
2355 case CmpInst::FCMP_FALSE: Opnd = I->getOperand(2); break;
2356 case CmpInst::FCMP_TRUE: Opnd = I->getOperand(1); break;
2357 }
2358 // No need for a select anymore - this is an unconditional move.
2359 if (Opnd) {
2360 Register OpReg = getRegForValue(Opnd);
2361 if (OpReg == 0)
2362 return false;
2363 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2364 Register ResultReg = createResultReg(RC);
2365 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2366 TII.get(TargetOpcode::COPY), ResultReg)
2367 .addReg(OpReg);
2368 updateValueMap(I, ResultReg);
2369 return true;
2370 }
2371 }
2372
2373 // First try to use real conditional move instructions.
2374 if (X86FastEmitCMoveSelect(RetVT, I))
2375 return true;
2376
2377 // Try to use a sequence of SSE instructions to simulate a conditional move.
2378 if (X86FastEmitSSESelect(RetVT, I))
2379 return true;
2380
2381 // Fall-back to pseudo conditional move instructions, which will be later
2382 // converted to control-flow.
2383 if (X86FastEmitPseudoSelect(RetVT, I))
2384 return true;
2385
2386 return false;
2387}
2388
2389// Common code for X86SelectSIToFP and X86SelectUIToFP.
2390bool X86FastISel::X86SelectIntToFP(const Instruction *I, bool IsSigned) {
2391 // The target-independent selection algorithm in FastISel already knows how
2392 // to select a SINT_TO_FP if the target is SSE but not AVX.
2393 // Early exit if the subtarget doesn't have AVX.
2394 // Unsigned conversion requires avx512.
2395 bool HasAVX512 = Subtarget->hasAVX512();
2396 if (!Subtarget->hasAVX() || (!IsSigned && !HasAVX512))
2397 return false;
2398
2399 // TODO: We could sign extend narrower types.
2400 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
2401 if (SrcVT != MVT::i32 && SrcVT != MVT::i64)
2402 return false;
2403
2404 // Select integer to float/double conversion.
2405 Register OpReg = getRegForValue(I->getOperand(0));
2406 if (OpReg == 0)
2407 return false;
2408
2409 unsigned Opcode;
2410
2411 static const uint16_t SCvtOpc[2][2][2] = {
2412 { { X86::VCVTSI2SSrr, X86::VCVTSI642SSrr },
2413 { X86::VCVTSI2SDrr, X86::VCVTSI642SDrr } },
2414 { { X86::VCVTSI2SSZrr, X86::VCVTSI642SSZrr },
2415 { X86::VCVTSI2SDZrr, X86::VCVTSI642SDZrr } },
2416 };
2417 static const uint16_t UCvtOpc[2][2] = {
2418 { X86::VCVTUSI2SSZrr, X86::VCVTUSI642SSZrr },
2419 { X86::VCVTUSI2SDZrr, X86::VCVTUSI642SDZrr },
2420 };
2421 bool Is64Bit = SrcVT == MVT::i64;
2422
2423 if (I->getType()->isDoubleTy()) {
2424 // s/uitofp int -> double
2425 Opcode = IsSigned ? SCvtOpc[HasAVX512][1][Is64Bit] : UCvtOpc[1][Is64Bit];
2426 } else if (I->getType()->isFloatTy()) {
2427 // s/uitofp int -> float
2428 Opcode = IsSigned ? SCvtOpc[HasAVX512][0][Is64Bit] : UCvtOpc[0][Is64Bit];
2429 } else
2430 return false;
2431
2432 MVT DstVT = TLI.getValueType(DL, I->getType()).getSimpleVT();
2433 const TargetRegisterClass *RC = TLI.getRegClassFor(DstVT);
2434 Register ImplicitDefReg = createResultReg(RC);
2435 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2436 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2437 Register ResultReg = fastEmitInst_rr(Opcode, RC, ImplicitDefReg, OpReg);
2438 updateValueMap(I, ResultReg);
2439 return true;
2440}
2441
2442bool X86FastISel::X86SelectSIToFP(const Instruction *I) {
2443 return X86SelectIntToFP(I, /*IsSigned*/true);
2444}
2445
2446bool X86FastISel::X86SelectUIToFP(const Instruction *I) {
2447 return X86SelectIntToFP(I, /*IsSigned*/false);
2448}
2449
2450// Helper method used by X86SelectFPExt and X86SelectFPTrunc.
2451bool X86FastISel::X86SelectFPExtOrFPTrunc(const Instruction *I,
2452 unsigned TargetOpc,
2453 const TargetRegisterClass *RC) {
2454 assert((I->getOpcode() == Instruction::FPExt ||
2455 I->getOpcode() == Instruction::FPTrunc) &&
2456 "Instruction must be an FPExt or FPTrunc!");
2457 bool HasAVX = Subtarget->hasAVX();
2458
2459 Register OpReg = getRegForValue(I->getOperand(0));
2460 if (OpReg == 0)
2461 return false;
2462
2463 unsigned ImplicitDefReg;
2464 if (HasAVX) {
2465 ImplicitDefReg = createResultReg(RC);
2466 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2467 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2468
2469 }
2470
2471 Register ResultReg = createResultReg(RC);
2473 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpc),
2474 ResultReg);
2475
2476 if (HasAVX)
2477 MIB.addReg(ImplicitDefReg);
2478
2479 MIB.addReg(OpReg);
2480 updateValueMap(I, ResultReg);
2481 return true;
2482}
2483
2484bool X86FastISel::X86SelectFPExt(const Instruction *I) {
2485 if (Subtarget->hasSSE2() && I->getType()->isDoubleTy() &&
2486 I->getOperand(0)->getType()->isFloatTy()) {
2487 bool HasAVX512 = Subtarget->hasAVX512();
2488 // fpext from float to double.
2489 unsigned Opc =
2490 HasAVX512 ? X86::VCVTSS2SDZrr
2491 : Subtarget->hasAVX() ? X86::VCVTSS2SDrr : X86::CVTSS2SDrr;
2492 return X86SelectFPExtOrFPTrunc(I, Opc, TLI.getRegClassFor(MVT::f64));
2493 }
2494
2495 return false;
2496}
2497
2498bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {
2499 if (Subtarget->hasSSE2() && I->getType()->isFloatTy() &&
2500 I->getOperand(0)->getType()->isDoubleTy()) {
2501 bool HasAVX512 = Subtarget->hasAVX512();
2502 // fptrunc from double to float.
2503 unsigned Opc =
2504 HasAVX512 ? X86::VCVTSD2SSZrr
2505 : Subtarget->hasAVX() ? X86::VCVTSD2SSrr : X86::CVTSD2SSrr;
2506 return X86SelectFPExtOrFPTrunc(I, Opc, TLI.getRegClassFor(MVT::f32));
2507 }
2508
2509 return false;
2510}
2511
2512bool X86FastISel::X86SelectTrunc(const Instruction *I) {
2513 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
2514 EVT DstVT = TLI.getValueType(DL, I->getType());
2515
2516 // This code only handles truncation to byte.
2517 if (DstVT != MVT::i8 && DstVT != MVT::i1)
2518 return false;
2519 if (!TLI.isTypeLegal(SrcVT))
2520 return false;
2521
2522 Register InputReg = getRegForValue(I->getOperand(0));
2523 if (!InputReg)
2524 // Unhandled operand. Halt "fast" selection and bail.
2525 return false;
2526
2527 if (SrcVT == MVT::i8) {
2528 // Truncate from i8 to i1; no code needed.
2529 updateValueMap(I, InputReg);
2530 return true;
2531 }
2532
2533 // Issue an extract_subreg.
2534 Register ResultReg = fastEmitInst_extractsubreg(MVT::i8, InputReg,
2535 X86::sub_8bit);
2536 if (!ResultReg)
2537 return false;
2538
2539 updateValueMap(I, ResultReg);
2540 return true;
2541}
2542
2543bool X86FastISel::IsMemcpySmall(uint64_t Len) {
2544 return Len <= (Subtarget->is64Bit() ? 32 : 16);
2545}
2546
2547bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM,
2548 X86AddressMode SrcAM, uint64_t Len) {
2549
2550 // Make sure we don't bloat code by inlining very large memcpy's.
2551 if (!IsMemcpySmall(Len))
2552 return false;
2553
2554 bool i64Legal = Subtarget->is64Bit();
2555
2556 // We don't care about alignment here since we just emit integer accesses.
2557 while (Len) {
2558 MVT VT;
2559 if (Len >= 8 && i64Legal)
2560 VT = MVT::i64;
2561 else if (Len >= 4)
2562 VT = MVT::i32;
2563 else if (Len >= 2)
2564 VT = MVT::i16;
2565 else
2566 VT = MVT::i8;
2567
2568 unsigned Reg;
2569 bool RV = X86FastEmitLoad(VT, SrcAM, nullptr, Reg);
2570 RV &= X86FastEmitStore(VT, Reg, DestAM);
2571 assert(RV && "Failed to emit load or store??");
2572 (void)RV;
2573
2574 unsigned Size = VT.getSizeInBits()/8;
2575 Len -= Size;
2576 DestAM.Disp += Size;
2577 SrcAM.Disp += Size;
2578 }
2579
2580 return true;
2581}
2582
2583bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
2584 // FIXME: Handle more intrinsics.
2585 switch (II->getIntrinsicID()) {
2586 default: return false;
2587 case Intrinsic::convert_from_fp16:
2588 case Intrinsic::convert_to_fp16: {
2589 if (Subtarget->useSoftFloat() || !Subtarget->hasF16C())
2590 return false;
2591
2592 const Value *Op = II->getArgOperand(0);
2593 Register InputReg = getRegForValue(Op);
2594 if (InputReg == 0)
2595 return false;
2596
2597 // F16C only allows converting from float to half and from half to float.
2598 bool IsFloatToHalf = II->getIntrinsicID() == Intrinsic::convert_to_fp16;
2599 if (IsFloatToHalf) {
2600 if (!Op->getType()->isFloatTy())
2601 return false;
2602 } else {
2603 if (!II->getType()->isFloatTy())
2604 return false;
2605 }
2606
2607 unsigned ResultReg = 0;
2608 const TargetRegisterClass *RC = TLI.getRegClassFor(MVT::v8i16);
2609 if (IsFloatToHalf) {
2610 // 'InputReg' is implicitly promoted from register class FR32 to
2611 // register class VR128 by method 'constrainOperandRegClass' which is
2612 // directly called by 'fastEmitInst_ri'.
2613 // Instruction VCVTPS2PHrr takes an extra immediate operand which is
2614 // used to provide rounding control: use MXCSR.RC, encoded as 0b100.
2615 // It's consistent with the other FP instructions, which are usually
2616 // controlled by MXCSR.
2617 unsigned Opc = Subtarget->hasVLX() ? X86::VCVTPS2PHZ128rr
2618 : X86::VCVTPS2PHrr;
2619 InputReg = fastEmitInst_ri(Opc, RC, InputReg, 4);
2620
2621 // Move the lower 32-bits of ResultReg to another register of class GR32.
2622 Opc = Subtarget->hasAVX512() ? X86::VMOVPDI2DIZrr
2623 : X86::VMOVPDI2DIrr;
2624 ResultReg = createResultReg(&X86::GR32RegClass);
2625 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
2626 .addReg(InputReg, RegState::Kill);
2627
2628 // The result value is in the lower 16-bits of ResultReg.
2629 unsigned RegIdx = X86::sub_16bit;
2630 ResultReg = fastEmitInst_extractsubreg(MVT::i16, ResultReg, RegIdx);
2631 } else {
2632 assert(Op->getType()->isIntegerTy(16) && "Expected a 16-bit integer!");
2633 // Explicitly zero-extend the input to 32-bit.
2634 InputReg = fastEmit_r(MVT::i16, MVT::i32, ISD::ZERO_EXTEND, InputReg);
2635
2636 // The following SCALAR_TO_VECTOR will be expanded into a VMOVDI2PDIrr.
2637 InputReg = fastEmit_r(MVT::i32, MVT::v4i32, ISD::SCALAR_TO_VECTOR,
2638 InputReg);
2639
2640 unsigned Opc = Subtarget->hasVLX() ? X86::VCVTPH2PSZ128rr
2641 : X86::VCVTPH2PSrr;
2642 InputReg = fastEmitInst_r(Opc, RC, InputReg);
2643
2644 // The result value is in the lower 32-bits of ResultReg.
2645 // Emit an explicit copy from register class VR128 to register class FR32.
2646 ResultReg = createResultReg(TLI.getRegClassFor(MVT::f32));
2647 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2648 TII.get(TargetOpcode::COPY), ResultReg)
2649 .addReg(InputReg, RegState::Kill);
2650 }
2651
2652 updateValueMap(II, ResultReg);
2653 return true;
2654 }
2655 case Intrinsic::frameaddress: {
2656 MachineFunction *MF = FuncInfo.MF;
2657 if (MF->getTarget().getMCAsmInfo()->usesWindowsCFI())
2658 return false;
2659
2661
2662 MVT VT;
2663 if (!isTypeLegal(RetTy, VT))
2664 return false;
2665
2666 unsigned Opc;
2667 const TargetRegisterClass *RC = nullptr;
2668
2669 switch (VT.SimpleTy) {
2670 default: llvm_unreachable("Invalid result type for frameaddress.");
2671 case MVT::i32: Opc = X86::MOV32rm; RC = &X86::GR32RegClass; break;
2672 case MVT::i64: Opc = X86::MOV64rm; RC = &X86::GR64RegClass; break;
2673 }
2674
2675 // This needs to be set before we call getPtrSizedFrameRegister, otherwise
2676 // we get the wrong frame register.
2677 MachineFrameInfo &MFI = MF->getFrameInfo();
2678 MFI.setFrameAddressIsTaken(true);
2679
2680 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2681 unsigned FrameReg = RegInfo->getPtrSizedFrameRegister(*MF);
2682 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
2683 (FrameReg == X86::EBP && VT == MVT::i32)) &&
2684 "Invalid Frame Register!");
2685
2686 // Always make a copy of the frame register to a vreg first, so that we
2687 // never directly reference the frame register (the TwoAddressInstruction-
2688 // Pass doesn't like that).
2689 Register SrcReg = createResultReg(RC);
2690 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2691 TII.get(TargetOpcode::COPY), SrcReg).addReg(FrameReg);
2692
2693 // Now recursively load from the frame address.
2694 // movq (%rbp), %rax
2695 // movq (%rax), %rax
2696 // movq (%rax), %rax
2697 // ...
2698 unsigned Depth = cast<ConstantInt>(II->getOperand(0))->getZExtValue();
2699 while (Depth--) {
2700 Register DestReg = createResultReg(RC);
2701 addDirectMem(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2702 TII.get(Opc), DestReg), SrcReg);
2703 SrcReg = DestReg;
2704 }
2705
2706 updateValueMap(II, SrcReg);
2707 return true;
2708 }
2709 case Intrinsic::memcpy: {
2710 const MemCpyInst *MCI = cast<MemCpyInst>(II);
2711 // Don't handle volatile or variable length memcpys.
2712 if (MCI->isVolatile())
2713 return false;
2714
2715 if (isa<ConstantInt>(MCI->getLength())) {
2716 // Small memcpy's are common enough that we want to do them
2717 // without a call if possible.
2718 uint64_t Len = cast<ConstantInt>(MCI->getLength())->getZExtValue();
2719 if (IsMemcpySmall(Len)) {
2720 X86AddressMode DestAM, SrcAM;
2721 if (!X86SelectAddress(MCI->getRawDest(), DestAM) ||
2722 !X86SelectAddress(MCI->getRawSource(), SrcAM))
2723 return false;
2724 TryEmitSmallMemcpy(DestAM, SrcAM, Len);
2725 return true;
2726 }
2727 }
2728
2729 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2730 if (!MCI->getLength()->getType()->isIntegerTy(SizeWidth))
2731 return false;
2732
2733 if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255)
2734 return false;
2735
2736 return lowerCallTo(II, "memcpy", II->arg_size() - 1);
2737 }
2738 case Intrinsic::memset: {
2739 const MemSetInst *MSI = cast<MemSetInst>(II);
2740
2741 if (MSI->isVolatile())
2742 return false;
2743
2744 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2745 if (!MSI->getLength()->getType()->isIntegerTy(SizeWidth))
2746 return false;
2747
2748 if (MSI->getDestAddressSpace() > 255)
2749 return false;
2750
2751 return lowerCallTo(II, "memset", II->arg_size() - 1);
2752 }
2753 case Intrinsic::stackprotector: {
2754 // Emit code to store the stack guard onto the stack.
2755 EVT PtrTy = TLI.getPointerTy(DL);
2756
2757 const Value *Op1 = II->getArgOperand(0); // The guard's value.
2758 const AllocaInst *Slot = cast<AllocaInst>(II->getArgOperand(1));
2759
2760 MFI.setStackProtectorIndex(FuncInfo.StaticAllocaMap[Slot]);
2761
2762 // Grab the frame index.
2763 X86AddressMode AM;
2764 if (!X86SelectAddress(Slot, AM)) return false;
2765 if (!X86FastEmitStore(PtrTy, Op1, AM)) return false;
2766 return true;
2767 }
2768 case Intrinsic::dbg_declare: {
2769 const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
2770 X86AddressMode AM;
2771 assert(DI->getAddress() && "Null address should be checked earlier!");
2772 if (!X86SelectAddress(DI->getAddress(), AM))
2773 return false;
2774 const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
2775 assert(DI->getVariable()->isValidLocationForIntrinsic(MIMD.getDL()) &&
2776 "Expected inlined-at fields to agree");
2777 addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II), AM)
2778 .addImm(0)
2779 .addMetadata(DI->getVariable())
2780 .addMetadata(DI->getExpression());
2781 return true;
2782 }
2783 case Intrinsic::trap: {
2784 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::TRAP));
2785 return true;
2786 }
2787 case Intrinsic::sqrt: {
2788 if (!Subtarget->hasSSE1())
2789 return false;
2790
2791 Type *RetTy = II->getCalledFunction()->getReturnType();
2792
2793 MVT VT;
2794 if (!isTypeLegal(RetTy, VT))
2795 return false;
2796
2797 // Unfortunately we can't use fastEmit_r, because the AVX version of FSQRT
2798 // is not generated by FastISel yet.
2799 // FIXME: Update this code once tablegen can handle it.
2800 static const uint16_t SqrtOpc[3][2] = {
2801 { X86::SQRTSSr, X86::SQRTSDr },
2802 { X86::VSQRTSSr, X86::VSQRTSDr },
2803 { X86::VSQRTSSZr, X86::VSQRTSDZr },
2804 };
2805 unsigned AVXLevel = Subtarget->hasAVX512() ? 2 :
2806 Subtarget->hasAVX() ? 1 :
2807 0;
2808 unsigned Opc;
2809 switch (VT.SimpleTy) {
2810 default: return false;
2811 case MVT::f32: Opc = SqrtOpc[AVXLevel][0]; break;
2812 case MVT::f64: Opc = SqrtOpc[AVXLevel][1]; break;
2813 }
2814
2815 const Value *SrcVal = II->getArgOperand(0);
2816 Register SrcReg = getRegForValue(SrcVal);
2817
2818 if (SrcReg == 0)
2819 return false;
2820
2821 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
2822 unsigned ImplicitDefReg = 0;
2823 if (AVXLevel > 0) {
2824 ImplicitDefReg = createResultReg(RC);
2825 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2826 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2827 }
2828
2829 Register ResultReg = createResultReg(RC);
2831 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc),
2832 ResultReg);
2833
2834 if (ImplicitDefReg)
2835 MIB.addReg(ImplicitDefReg);
2836
2837 MIB.addReg(SrcReg);
2838
2839 updateValueMap(II, ResultReg);
2840 return true;
2841 }
2842 case Intrinsic::sadd_with_overflow:
2843 case Intrinsic::uadd_with_overflow:
2844 case Intrinsic::ssub_with_overflow:
2845 case Intrinsic::usub_with_overflow:
2846 case Intrinsic::smul_with_overflow:
2847 case Intrinsic::umul_with_overflow: {
2848 // This implements the basic lowering of the xalu with overflow intrinsics
2849 // into add/sub/mul followed by either seto or setb.
2850 const Function *Callee = II->getCalledFunction();
2851 auto *Ty = cast<StructType>(Callee->getReturnType());
2852 Type *RetTy = Ty->getTypeAtIndex(0U);
2853 assert(Ty->getTypeAtIndex(1)->isIntegerTy() &&
2854 Ty->getTypeAtIndex(1)->getScalarSizeInBits() == 1 &&
2855 "Overflow value expected to be an i1");
2856
2857 MVT VT;
2858 if (!isTypeLegal(RetTy, VT))
2859 return false;
2860
2861 if (VT < MVT::i8 || VT > MVT::i64)
2862 return false;
2863
2864 const Value *LHS = II->getArgOperand(0);
2865 const Value *RHS = II->getArgOperand(1);
2866
2867 // Canonicalize immediate to the RHS.
2868 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) && II->isCommutative())
2869 std::swap(LHS, RHS);
2870
2871 unsigned BaseOpc, CondCode;
2872 switch (II->getIntrinsicID()) {
2873 default: llvm_unreachable("Unexpected intrinsic!");
2874 case Intrinsic::sadd_with_overflow:
2875 BaseOpc = ISD::ADD; CondCode = X86::COND_O; break;
2876 case Intrinsic::uadd_with_overflow:
2877 BaseOpc = ISD::ADD; CondCode = X86::COND_B; break;
2878 case Intrinsic::ssub_with_overflow:
2879 BaseOpc = ISD::SUB; CondCode = X86::COND_O; break;
2880 case Intrinsic::usub_with_overflow:
2881 BaseOpc = ISD::SUB; CondCode = X86::COND_B; break;
2882 case Intrinsic::smul_with_overflow:
2883 BaseOpc = X86ISD::SMUL; CondCode = X86::COND_O; break;
2884 case Intrinsic::umul_with_overflow:
2885 BaseOpc = X86ISD::UMUL; CondCode = X86::COND_O; break;
2886 }
2887
2888 Register LHSReg = getRegForValue(LHS);
2889 if (LHSReg == 0)
2890 return false;
2891
2892 unsigned ResultReg = 0;
2893 // Check if we have an immediate version.
2894 if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {
2895 static const uint16_t Opc[2][4] = {
2896 { X86::INC8r, X86::INC16r, X86::INC32r, X86::INC64r },
2897 { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }
2898 };
2899
2900 if (CI->isOne() && (BaseOpc == ISD::ADD || BaseOpc == ISD::SUB) &&
2901 CondCode == X86::COND_O) {
2902 // We can use INC/DEC.
2903 ResultReg = createResultReg(TLI.getRegClassFor(VT));
2904 bool IsDec = BaseOpc == ISD::SUB;
2905 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2906 TII.get(Opc[IsDec][VT.SimpleTy-MVT::i8]), ResultReg)
2907 .addReg(LHSReg);
2908 } else
2909 ResultReg = fastEmit_ri(VT, VT, BaseOpc, LHSReg, CI->getZExtValue());
2910 }
2911
2912 unsigned RHSReg;
2913 if (!ResultReg) {
2914 RHSReg = getRegForValue(RHS);
2915 if (RHSReg == 0)
2916 return false;
2917 ResultReg = fastEmit_rr(VT, VT, BaseOpc, LHSReg, RHSReg);
2918 }
2919
2920 // FastISel doesn't have a pattern for all X86::MUL*r and X86::IMUL*r. Emit
2921 // it manually.
2922 if (BaseOpc == X86ISD::UMUL && !ResultReg) {
2923 static const uint16_t MULOpc[] =
2924 { X86::MUL8r, X86::MUL16r, X86::MUL32r, X86::MUL64r };
2925 static const MCPhysReg Reg[] = { X86::AL, X86::AX, X86::EAX, X86::RAX };
2926 // First copy the first operand into RAX, which is an implicit input to
2927 // the X86::MUL*r instruction.
2928 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2929 TII.get(TargetOpcode::COPY), Reg[VT.SimpleTy-MVT::i8])
2930 .addReg(LHSReg);
2931 ResultReg = fastEmitInst_r(MULOpc[VT.SimpleTy-MVT::i8],
2932 TLI.getRegClassFor(VT), RHSReg);
2933 } else if (BaseOpc == X86ISD::SMUL && !ResultReg) {
2934 static const uint16_t MULOpc[] =
2935 { X86::IMUL8r, X86::IMUL16rr, X86::IMUL32rr, X86::IMUL64rr };
2936 if (VT == MVT::i8) {
2937 // Copy the first operand into AL, which is an implicit input to the
2938 // X86::IMUL8r instruction.
2939 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2940 TII.get(TargetOpcode::COPY), X86::AL)
2941 .addReg(LHSReg);
2942 ResultReg = fastEmitInst_r(MULOpc[0], TLI.getRegClassFor(VT), RHSReg);
2943 } else
2944 ResultReg = fastEmitInst_rr(MULOpc[VT.SimpleTy-MVT::i8],
2945 TLI.getRegClassFor(VT), LHSReg, RHSReg);
2946 }
2947
2948 if (!ResultReg)
2949 return false;
2950
2951 // Assign to a GPR since the overflow return value is lowered to a SETcc.
2952 Register ResultReg2 = createResultReg(&X86::GR8RegClass);
2953 assert((ResultReg+1) == ResultReg2 && "Nonconsecutive result registers.");
2954 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
2955 ResultReg2).addImm(CondCode);
2956
2957 updateValueMap(II, ResultReg, 2);
2958 return true;
2959 }
2960 case Intrinsic::x86_sse_cvttss2si:
2961 case Intrinsic::x86_sse_cvttss2si64:
2962 case Intrinsic::x86_sse2_cvttsd2si:
2963 case Intrinsic::x86_sse2_cvttsd2si64: {
2964 bool IsInputDouble;
2965 switch (II->getIntrinsicID()) {
2966 default: llvm_unreachable("Unexpected intrinsic.");
2967 case Intrinsic::x86_sse_cvttss2si:
2968 case Intrinsic::x86_sse_cvttss2si64:
2969 if (!Subtarget->hasSSE1())
2970 return false;
2971 IsInputDouble = false;
2972 break;
2973 case Intrinsic::x86_sse2_cvttsd2si:
2974 case Intrinsic::x86_sse2_cvttsd2si64:
2975 if (!Subtarget->hasSSE2())
2976 return false;
2977 IsInputDouble = true;
2978 break;
2979 }
2980
2981 Type *RetTy = II->getCalledFunction()->getReturnType();
2982 MVT VT;
2983 if (!isTypeLegal(RetTy, VT))
2984 return false;
2985
2986 static const uint16_t CvtOpc[3][2][2] = {
2987 { { X86::CVTTSS2SIrr, X86::CVTTSS2SI64rr },
2988 { X86::CVTTSD2SIrr, X86::CVTTSD2SI64rr } },
2989 { { X86::VCVTTSS2SIrr, X86::VCVTTSS2SI64rr },
2990 { X86::VCVTTSD2SIrr, X86::VCVTTSD2SI64rr } },
2991 { { X86::VCVTTSS2SIZrr, X86::VCVTTSS2SI64Zrr },
2992 { X86::VCVTTSD2SIZrr, X86::VCVTTSD2SI64Zrr } },
2993 };
2994 unsigned AVXLevel = Subtarget->hasAVX512() ? 2 :
2995 Subtarget->hasAVX() ? 1 :
2996 0;
2997 unsigned Opc;
2998 switch (VT.SimpleTy) {
2999 default: llvm_unreachable("Unexpected result type.");
3000 case MVT::i32: Opc = CvtOpc[AVXLevel][IsInputDouble][0]; break;
3001 case MVT::i64: Opc = CvtOpc[AVXLevel][IsInputDouble][1]; break;
3002 }
3003
3004 // Check if we can fold insertelement instructions into the convert.
3005 const Value *Op = II->getArgOperand(0);
3006 while (auto *IE = dyn_cast<InsertElementInst>(Op)) {
3007 const Value *Index = IE->getOperand(2);
3008 if (!isa<ConstantInt>(Index))
3009 break;
3010 unsigned Idx = cast<ConstantInt>(Index)->getZExtValue();
3011
3012 if (Idx == 0) {
3013 Op = IE->getOperand(1);
3014 break;
3015 }
3016 Op = IE->getOperand(0);
3017 }
3018
3019 Register Reg = getRegForValue(Op);
3020 if (Reg == 0)
3021 return false;
3022
3023 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3024 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
3025 .addReg(Reg);
3026
3027 updateValueMap(II, ResultReg);
3028 return true;
3029 }
3030 case Intrinsic::x86_sse42_crc32_32_8:
3031 case Intrinsic::x86_sse42_crc32_32_16:
3032 case Intrinsic::x86_sse42_crc32_32_32:
3033 case Intrinsic::x86_sse42_crc32_64_64: {
3034 if (!Subtarget->hasCRC32())
3035 return false;
3036
3037 Type *RetTy = II->getCalledFunction()->getReturnType();
3038
3039 MVT VT;
3040 if (!isTypeLegal(RetTy, VT))
3041 return false;
3042
3043 unsigned Opc;
3044 const TargetRegisterClass *RC = nullptr;
3045
3046 switch (II->getIntrinsicID()) {
3047 default:
3048 llvm_unreachable("Unexpected intrinsic.");
3049#define GET_EGPR_IF_ENABLED(OPC) Subtarget->hasEGPR() ? OPC##_EVEX : OPC
3050 case Intrinsic::x86_sse42_crc32_32_8:
3051 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r8);
3052 RC = &X86::GR32RegClass;
3053 break;
3054 case Intrinsic::x86_sse42_crc32_32_16:
3055 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r16);
3056 RC = &X86::GR32RegClass;
3057 break;
3058 case Intrinsic::x86_sse42_crc32_32_32:
3059 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r32);
3060 RC = &X86::GR32RegClass;
3061 break;
3062 case Intrinsic::x86_sse42_crc32_64_64:
3063 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r64r64);
3064 RC = &X86::GR64RegClass;
3065 break;
3066#undef GET_EGPR_IF_ENABLED
3067 }
3068
3069 const Value *LHS = II->getArgOperand(0);
3070 const Value *RHS = II->getArgOperand(1);
3071
3072 Register LHSReg = getRegForValue(LHS);
3073 Register RHSReg = getRegForValue(RHS);
3074 if (!LHSReg || !RHSReg)
3075 return false;
3076
3077 Register ResultReg = fastEmitInst_rr(Opc, RC, LHSReg, RHSReg);
3078 if (!ResultReg)
3079 return false;
3080
3081 updateValueMap(II, ResultReg);
3082 return true;
3083 }
3084 }
3085}
3086
3087bool X86FastISel::fastLowerArguments() {
3088 if (!FuncInfo.CanLowerReturn)
3089 return false;
3090
3091 const Function *F = FuncInfo.Fn;
3092 if (F->isVarArg())
3093 return false;
3094
3095 CallingConv::ID CC = F->getCallingConv();
3096 if (CC != CallingConv::C)
3097 return false;
3098
3099 if (Subtarget->isCallingConvWin64(CC))
3100 return false;
3101
3102 if (!Subtarget->is64Bit())
3103 return false;
3104
3105 if (Subtarget->useSoftFloat())
3106 return false;
3107
3108 // Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments.
3109 unsigned GPRCnt = 0;
3110 unsigned FPRCnt = 0;
3111 for (auto const &Arg : F->args()) {
3112 if (Arg.hasAttribute(Attribute::ByVal) ||
3113 Arg.hasAttribute(Attribute::InReg) ||
3114 Arg.hasAttribute(Attribute::StructRet) ||
3115 Arg.hasAttribute(Attribute::SwiftSelf) ||
3116 Arg.hasAttribute(Attribute::SwiftAsync) ||
3117 Arg.hasAttribute(Attribute::SwiftError) ||
3118 Arg.hasAttribute(Attribute::Nest))
3119 return false;
3120
3121 Type *ArgTy = Arg.getType();
3122 if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy())
3123 return false;
3124
3125 EVT ArgVT = TLI.getValueType(DL, ArgTy);
3126 if (!ArgVT.isSimple()) return false;
3127 switch (ArgVT.getSimpleVT().SimpleTy) {
3128 default: return false;
3129 case MVT::i32:
3130 case MVT::i64:
3131 ++GPRCnt;
3132 break;
3133 case MVT::f32:
3134 case MVT::f64:
3135 if (!Subtarget->hasSSE1())
3136 return false;
3137 ++FPRCnt;
3138 break;
3139 }
3140
3141 if (GPRCnt > 6)
3142 return false;
3143
3144 if (FPRCnt > 8)
3145 return false;
3146 }
3147
3148 static const MCPhysReg GPR32ArgRegs[] = {
3149 X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
3150 };
3151 static const MCPhysReg GPR64ArgRegs[] = {
3152 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8 , X86::R9
3153 };
3154 static const MCPhysReg XMMArgRegs[] = {
3155 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3156 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3157 };
3158
3159 unsigned GPRIdx = 0;
3160 unsigned FPRIdx = 0;
3161 for (auto const &Arg : F->args()) {
3162 MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
3163 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
3164 unsigned SrcReg;
3165 switch (VT.SimpleTy) {
3166 default: llvm_unreachable("Unexpected value type.");
3167 case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++]; break;
3168 case MVT::i64: SrcReg = GPR64ArgRegs[GPRIdx++]; break;
3169 case MVT::f32: [[fallthrough]];
3170 case MVT::f64: SrcReg = XMMArgRegs[FPRIdx++]; break;
3171 }
3172 Register DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
3173 // FIXME: Unfortunately it's necessary to emit a copy from the livein copy.
3174 // Without this, EmitLiveInCopies may eliminate the livein if its only
3175 // use is a bitcast (which isn't turned into an instruction).
3176 Register ResultReg = createResultReg(RC);
3177 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3178 TII.get(TargetOpcode::COPY), ResultReg)
3179 .addReg(DstReg, getKillRegState(true));
3180 updateValueMap(&Arg, ResultReg);
3181 }
3182 return true;
3183}
3184
3185static unsigned computeBytesPoppedByCalleeForSRet(const X86Subtarget *Subtarget,
3187 const CallBase *CB) {
3188 if (Subtarget->is64Bit())
3189 return 0;
3190 if (Subtarget->getTargetTriple().isOSMSVCRT())
3191 return 0;
3192 if (CC == CallingConv::Fast || CC == CallingConv::GHC ||
3195 return 0;
3196
3197 if (CB)
3198 if (CB->arg_empty() || !CB->paramHasAttr(0, Attribute::StructRet) ||
3199 CB->paramHasAttr(0, Attribute::InReg) || Subtarget->isTargetMCU())
3200 return 0;
3201
3202 return 4;
3203}
3204
3205bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3206 auto &OutVals = CLI.OutVals;
3207 auto &OutFlags = CLI.OutFlags;
3208 auto &OutRegs = CLI.OutRegs;
3209 auto &Ins = CLI.Ins;
3210 auto &InRegs = CLI.InRegs;
3211 CallingConv::ID CC = CLI.CallConv;
3212 bool &IsTailCall = CLI.IsTailCall;
3213 bool IsVarArg = CLI.IsVarArg;
3214 const Value *Callee = CLI.Callee;
3215 MCSymbol *Symbol = CLI.Symbol;
3216 const auto *CB = CLI.CB;
3217
3218 bool Is64Bit = Subtarget->is64Bit();
3219 bool IsWin64 = Subtarget->isCallingConvWin64(CC);
3220
3221 // Call / invoke instructions with NoCfCheck attribute require special
3222 // handling.
3223 if (CB && CB->doesNoCfCheck())
3224 return false;
3225
3226 // Functions with no_caller_saved_registers that need special handling.
3227 if ((CB && isa<CallInst>(CB) && CB->hasFnAttr("no_caller_saved_registers")))
3228 return false;
3229
3230 // Functions with no_callee_saved_registers that need special handling.
3231 if ((CB && CB->hasFnAttr("no_callee_saved_registers")))
3232 return false;
3233
3234 // Indirect calls with CFI checks need special handling.
3235 if (CB && CB->isIndirectCall() && CB->getOperandBundle(LLVMContext::OB_kcfi))
3236 return false;
3237
3238 // Functions using thunks for indirect calls need to use SDISel.
3239 if (Subtarget->useIndirectThunkCalls())
3240 return false;
3241
3242 // Handle only C and fastcc calling conventions for now.
3243 switch (CC) {
3244 default: return false;
3245 case CallingConv::C:
3246 case CallingConv::Fast:
3247 case CallingConv::Tail:
3248 case CallingConv::Swift:
3253 case CallingConv::Win64:
3256 break;
3257 }
3258
3259 // Allow SelectionDAG isel to handle tail calls.
3260 if (IsTailCall)
3261 return false;
3262
3263 // fastcc with -tailcallopt is intended to provide a guaranteed
3264 // tail call optimization. Fastisel doesn't know how to do that.
3265 if ((CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt) ||
3267 return false;
3268
3269 // Don't know how to handle Win64 varargs yet. Nothing special needed for
3270 // x86-32. Special handling for x86-64 is implemented.
3271 if (IsVarArg && IsWin64)
3272 return false;
3273
3274 // Don't know about inalloca yet.
3275 if (CLI.CB && CLI.CB->hasInAllocaArgument())
3276 return false;
3277
3278 for (auto Flag : CLI.OutFlags)
3279 if (Flag.isSwiftError() || Flag.isPreallocated())
3280 return false;
3281
3282 SmallVector<MVT, 16> OutVTs;
3284
3285 // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra
3286 // instruction. This is safe because it is common to all FastISel supported
3287 // calling conventions on x86.
3288 for (int i = 0, e = OutVals.size(); i != e; ++i) {
3289 Value *&Val = OutVals[i];
3290 ISD::ArgFlagsTy Flags = OutFlags[i];
3291 if (auto *CI = dyn_cast<ConstantInt>(Val)) {
3292 if (CI->getBitWidth() < 32) {
3293 if (Flags.isSExt())
3294 Val = ConstantInt::get(CI->getContext(), CI->getValue().sext(32));
3295 else
3296 Val = ConstantInt::get(CI->getContext(), CI->getValue().zext(32));
3297 }
3298 }
3299
3300 // Passing bools around ends up doing a trunc to i1 and passing it.
3301 // Codegen this as an argument + "and 1".
3302 MVT VT;
3303 auto *TI = dyn_cast<TruncInst>(Val);
3304 unsigned ResultReg;
3305 if (TI && TI->getType()->isIntegerTy(1) && CLI.CB &&
3306 (TI->getParent() == CLI.CB->getParent()) && TI->hasOneUse()) {
3307 Value *PrevVal = TI->getOperand(0);
3308 ResultReg = getRegForValue(PrevVal);
3309
3310 if (!ResultReg)
3311 return false;
3312
3313 if (!isTypeLegal(PrevVal->getType(), VT))
3314 return false;
3315
3316 ResultReg = fastEmit_ri(VT, VT, ISD::AND, ResultReg, 1);
3317 } else {
3318 if (!isTypeLegal(Val->getType(), VT) ||
3319 (VT.isVector() && VT.getVectorElementType() == MVT::i1))
3320 return false;
3321 ResultReg = getRegForValue(Val);
3322 }
3323
3324 if (!ResultReg)
3325 return false;
3326
3327 ArgRegs.push_back(ResultReg);
3328 OutVTs.push_back(VT);
3329 }
3330
3331 // Analyze operands of the call, assigning locations to each operand.
3333 CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext());
3334
3335 // Allocate shadow area for Win64
3336 if (IsWin64)
3337 CCInfo.AllocateStack(32, Align(8));
3338
3339 CCInfo.AnalyzeCallOperands(OutVTs, OutFlags, CC_X86);
3340
3341 // Get a count of how many bytes are to be pushed on the stack.
3342 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
3343
3344 // Issue CALLSEQ_START
3345 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
3346 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackDown))
3347 .addImm(NumBytes).addImm(0).addImm(0);
3348
3349 // Walk the register/memloc assignments, inserting copies/loads.
3350 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3351 for (const CCValAssign &VA : ArgLocs) {
3352 const Value *ArgVal = OutVals[VA.getValNo()];
3353 MVT ArgVT = OutVTs[VA.getValNo()];
3354
3355 if (ArgVT == MVT::x86mmx)
3356 return false;
3357
3358 unsigned ArgReg = ArgRegs[VA.getValNo()];
3359
3360 // Promote the value if needed.
3361 switch (VA.getLocInfo()) {
3362 case CCValAssign::Full: break;
3363 case CCValAssign::SExt: {
3364 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3365 "Unexpected extend");
3366
3367 if (ArgVT == MVT::i1)
3368 return false;
3369
3370 bool Emitted = X86FastEmitExtend(ISD::SIGN_EXTEND, VA.getLocVT(), ArgReg,
3371 ArgVT, ArgReg);
3372 assert(Emitted && "Failed to emit a sext!"); (void)Emitted;
3373 ArgVT = VA.getLocVT();
3374 break;
3375 }
3376 case CCValAssign::ZExt: {
3377 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3378 "Unexpected extend");
3379
3380 // Handle zero-extension from i1 to i8, which is common.
3381 if (ArgVT == MVT::i1) {
3382 // Set the high bits to zero.
3383 ArgReg = fastEmitZExtFromI1(MVT::i8, ArgReg);
3384 ArgVT = MVT::i8;
3385
3386 if (ArgReg == 0)
3387 return false;
3388 }
3389
3390 bool Emitted = X86FastEmitExtend(ISD::ZERO_EXTEND, VA.getLocVT(), ArgReg,
3391 ArgVT, ArgReg);
3392 assert(Emitted && "Failed to emit a zext!"); (void)Emitted;
3393 ArgVT = VA.getLocVT();
3394 break;
3395 }
3396 case CCValAssign::AExt: {
3397 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3398 "Unexpected extend");
3399 bool Emitted = X86FastEmitExtend(ISD::ANY_EXTEND, VA.getLocVT(), ArgReg,
3400 ArgVT, ArgReg);
3401 if (!Emitted)
3402 Emitted = X86FastEmitExtend(ISD::ZERO_EXTEND, VA.getLocVT(), ArgReg,
3403 ArgVT, ArgReg);
3404 if (!Emitted)
3405 Emitted = X86FastEmitExtend(ISD::SIGN_EXTEND, VA.getLocVT(), ArgReg,
3406 ArgVT, ArgReg);
3407
3408 assert(Emitted && "Failed to emit a aext!"); (void)Emitted;
3409 ArgVT = VA.getLocVT();
3410 break;
3411 }
3412 case CCValAssign::BCvt: {
3413 ArgReg = fastEmit_r(ArgVT, VA.getLocVT(), ISD::BITCAST, ArgReg);
3414 assert(ArgReg && "Failed to emit a bitcast!");
3415 ArgVT = VA.getLocVT();
3416 break;
3417 }
3418 case CCValAssign::VExt:
3419 // VExt has not been implemented, so this should be impossible to reach
3420 // for now. However, fallback to Selection DAG isel once implemented.
3421 return false;
3425 case CCValAssign::FPExt:
3426 case CCValAssign::Trunc:
3427 llvm_unreachable("Unexpected loc info!");
3429 // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully
3430 // support this.
3431 return false;
3432 }
3433
3434 if (VA.isRegLoc()) {
3435 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3436 TII.get(TargetOpcode::COPY), VA.getLocReg()).addReg(ArgReg);
3437 OutRegs.push_back(VA.getLocReg());
3438 } else {
3439 assert(VA.isMemLoc() && "Unknown value location!");
3440
3441 // Don't emit stores for undef values.
3442 if (isa<UndefValue>(ArgVal))
3443 continue;
3444
3445 unsigned LocMemOffset = VA.getLocMemOffset();
3446 X86AddressMode AM;
3447 AM.Base.Reg = RegInfo->getStackRegister();
3448 AM.Disp = LocMemOffset;
3449 ISD::ArgFlagsTy Flags = OutFlags[VA.getValNo()];
3450 Align Alignment = DL.getABITypeAlign(ArgVal->getType());
3451 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3452 MachinePointerInfo::getStack(*FuncInfo.MF, LocMemOffset),
3453 MachineMemOperand::MOStore, ArgVT.getStoreSize(), Alignment);
3454 if (Flags.isByVal()) {
3455 X86AddressMode SrcAM;
3456 SrcAM.Base.Reg = ArgReg;
3457 if (!TryEmitSmallMemcpy(AM, SrcAM, Flags.getByValSize()))
3458 return false;
3459 } else if (isa<ConstantInt>(ArgVal) || isa<ConstantPointerNull>(ArgVal)) {
3460 // If this is a really simple value, emit this with the Value* version
3461 // of X86FastEmitStore. If it isn't simple, we don't want to do this,
3462 // as it can cause us to reevaluate the argument.
3463 if (!X86FastEmitStore(ArgVT, ArgVal, AM, MMO))
3464 return false;
3465 } else {
3466 if (!X86FastEmitStore(ArgVT, ArgReg, AM, MMO))
3467 return false;
3468 }
3469 }
3470 }
3471
3472 // ELF / PIC requires GOT in the EBX register before function calls via PLT
3473 // GOT pointer.
3474 if (Subtarget->isPICStyleGOT()) {
3475 unsigned Base = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3476 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3477 TII.get(TargetOpcode::COPY), X86::EBX).addReg(Base);
3478 }
3479
3480 if (Is64Bit && IsVarArg && !IsWin64) {
3481 // From AMD64 ABI document:
3482 // For calls that may call functions that use varargs or stdargs
3483 // (prototype-less calls or calls to functions containing ellipsis (...) in
3484 // the declaration) %al is used as hidden argument to specify the number
3485 // of SSE registers used. The contents of %al do not need to match exactly
3486 // the number of registers, but must be an ubound on the number of SSE
3487 // registers used and is in the range 0 - 8 inclusive.
3488
3489 // Count the number of XMM registers allocated.
3490 static const MCPhysReg XMMArgRegs[] = {
3491 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3492 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3493 };
3494 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);
3495 assert((Subtarget->hasSSE1() || !NumXMMRegs)
3496 && "SSE registers cannot be used when SSE is disabled");
3497 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV8ri),
3498 X86::AL).addImm(NumXMMRegs);
3499 }
3500
3501 // Materialize callee address in a register. FIXME: GV address can be
3502 // handled with a CALLpcrel32 instead.
3503 X86AddressMode CalleeAM;
3504 if (!X86SelectCallAddress(Callee, CalleeAM))
3505 return false;
3506
3507 unsigned CalleeOp = 0;
3508 const GlobalValue *GV = nullptr;
3509 if (CalleeAM.GV != nullptr) {
3510 GV = CalleeAM.GV;
3511 } else if (CalleeAM.Base.Reg != 0) {
3512 CalleeOp = CalleeAM.Base.Reg;
3513 } else
3514 return false;
3515
3516 // Issue the call.
3518 if (CalleeOp) {
3519 // Register-indirect call.
3520 unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;
3521 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(CallOpc))
3522 .addReg(CalleeOp);
3523 } else {
3524 // Direct call.
3525 assert(GV && "Not a direct call");
3526 // See if we need any target-specific flags on the GV operand.
3527 unsigned char OpFlags = Subtarget->classifyGlobalFunctionReference(GV);
3528 if (OpFlags == X86II::MO_PLT && !Is64Bit &&
3529 TM.getRelocationModel() == Reloc::Static && isa<Function>(GV) &&
3530 cast<Function>(GV)->isIntrinsic())
3531 OpFlags = X86II::MO_NO_FLAG;
3532
3533 // This will be a direct call, or an indirect call through memory for
3534 // NonLazyBind calls or dllimport calls.
3535 bool NeedLoad = OpFlags == X86II::MO_DLLIMPORT ||
3536 OpFlags == X86II::MO_GOTPCREL ||
3537 OpFlags == X86II::MO_GOTPCREL_NORELAX ||
3538 OpFlags == X86II::MO_COFFSTUB;
3539 unsigned CallOpc = NeedLoad
3540 ? (Is64Bit ? X86::CALL64m : X86::CALL32m)
3541 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32);
3542
3543 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(CallOpc));
3544 if (NeedLoad)
3545 MIB.addReg(Is64Bit ? X86::RIP : 0).addImm(1).addReg(0);
3546 if (Symbol)
3547 MIB.addSym(Symbol, OpFlags);
3548 else
3549 MIB.addGlobalAddress(GV, 0, OpFlags);
3550 if (NeedLoad)
3551 MIB.addReg(0);
3552 }
3553
3554 // Add a register mask operand representing the call-preserved registers.
3555 // Proper defs for return values will be added by setPhysRegsDeadExcept().
3556 MIB.addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC));
3557
3558 // Add an implicit use GOT pointer in EBX.
3559 if (Subtarget->isPICStyleGOT())
3560 MIB.addReg(X86::EBX, RegState::Implicit);
3561
3562 if (Is64Bit && IsVarArg && !IsWin64)
3563 MIB.addReg(X86::AL, RegState::Implicit);
3564
3565 // Add implicit physical register uses to the call.
3566 for (auto Reg : OutRegs)
3567 MIB.addReg(Reg, RegState::Implicit);
3568
3569 // Issue CALLSEQ_END
3570 unsigned NumBytesForCalleeToPop =
3571 X86::isCalleePop(CC, Subtarget->is64Bit(), IsVarArg,
3572 TM.Options.GuaranteedTailCallOpt)
3573 ? NumBytes // Callee pops everything.
3574 : computeBytesPoppedByCalleeForSRet(Subtarget, CC, CLI.CB);
3575 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
3576 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackUp))
3577 .addImm(NumBytes).addImm(NumBytesForCalleeToPop);
3578
3579 // Now handle call return values.
3581 CCState CCRetInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs,
3582 CLI.RetTy->getContext());
3583 CCRetInfo.AnalyzeCallResult(Ins, RetCC_X86);
3584
3585 // Copy all of the result registers out of their specified physreg.
3586 Register ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
3587 for (unsigned i = 0; i != RVLocs.size(); ++i) {
3588 CCValAssign &VA = RVLocs[i];
3589 EVT CopyVT = VA.getValVT();
3590 unsigned CopyReg = ResultReg + i;
3591 Register SrcReg = VA.getLocReg();
3592
3593 // If this is x86-64, and we disabled SSE, we can't return FP values
3594 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
3595 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
3596 report_fatal_error("SSE register return with SSE disabled");
3597 }
3598
3599 // If we prefer to use the value in xmm registers, copy it out as f80 and
3600 // use a truncate to move it from fp stack reg to xmm reg.
3601 if ((SrcReg == X86::FP0 || SrcReg == X86::FP1) &&
3602 isScalarFPTypeInSSEReg(VA.getValVT())) {
3603 CopyVT = MVT::f80;
3604 CopyReg = createResultReg(&X86::RFP80RegClass);
3605 }
3606
3607 // Copy out the result.
3608 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3609 TII.get(TargetOpcode::COPY), CopyReg).addReg(SrcReg);
3610 InRegs.push_back(VA.getLocReg());
3611
3612 // Round the f80 to the right size, which also moves it to the appropriate
3613 // xmm register. This is accomplished by storing the f80 value in memory
3614 // and then loading it back.
3615 if (CopyVT != VA.getValVT()) {
3616 EVT ResVT = VA.getValVT();
3617 unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;
3618 unsigned MemSize = ResVT.getSizeInBits()/8;
3619 int FI = MFI.CreateStackObject(MemSize, Align(MemSize), false);
3620 addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3621 TII.get(Opc)), FI)
3622 .addReg(CopyReg);
3623 Opc = ResVT == MVT::f32 ? X86::MOVSSrm_alt : X86::MOVSDrm_alt;
3624 addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3625 TII.get(Opc), ResultReg + i), FI);
3626 }
3627 }
3628
3629 CLI.ResultReg = ResultReg;
3630 CLI.NumResultRegs = RVLocs.size();
3631 CLI.Call = MIB;
3632
3633 return true;
3634}
3635
3636bool
3637X86FastISel::fastSelectInstruction(const Instruction *I) {
3638 switch (I->getOpcode()) {
3639 default: break;
3640 case Instruction::Load:
3641 return X86SelectLoad(I);
3642 case Instruction::Store:
3643 return X86SelectStore(I);
3644 case Instruction::Ret:
3645 return X86SelectRet(I);
3646 case Instruction::ICmp:
3647 case Instruction::FCmp:
3648 return X86SelectCmp(I);
3649 case Instruction::ZExt:
3650 return X86SelectZExt(I);
3651 case Instruction::SExt:
3652 return X86SelectSExt(I);
3653 case Instruction::Br:
3654 return X86SelectBranch(I);
3655 case Instruction::LShr:
3656 case Instruction::AShr:
3657 case Instruction::Shl:
3658 return X86SelectShift(I);
3659 case Instruction::SDiv:
3660 case Instruction::UDiv:
3661 case Instruction::SRem:
3662 case Instruction::URem:
3663 return X86SelectDivRem(I);
3664 case Instruction::Select:
3665 return X86SelectSelect(I);
3666 case Instruction::Trunc:
3667 return X86SelectTrunc(I);
3668 case Instruction::FPExt:
3669 return X86SelectFPExt(I);
3670 case Instruction::FPTrunc:
3671 return X86SelectFPTrunc(I);
3672 case Instruction::SIToFP:
3673 return X86SelectSIToFP(I);
3674 case Instruction::UIToFP:
3675 return X86SelectUIToFP(I);
3676 case Instruction::IntToPtr: // Deliberate fall-through.
3677 case Instruction::PtrToInt: {
3678 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
3679 EVT DstVT = TLI.getValueType(DL, I->getType());
3680 if (DstVT.bitsGT(SrcVT))
3681 return X86SelectZExt(I);
3682 if (DstVT.bitsLT(SrcVT))
3683 return X86SelectTrunc(I);
3684 Register Reg = getRegForValue(I->getOperand(0));
3685 if (Reg == 0) return false;
3686 updateValueMap(I, Reg);
3687 return true;
3688 }
3689 case Instruction::BitCast: {
3690 // Select SSE2/AVX bitcasts between 128/256/512 bit vector types.
3691 if (!Subtarget->hasSSE2())
3692 return false;
3693
3694 MVT SrcVT, DstVT;
3695 if (!isTypeLegal(I->getOperand(0)->getType(), SrcVT) ||
3696 !isTypeLegal(I->getType(), DstVT))
3697 return false;
3698
3699 // Only allow vectors that use xmm/ymm/zmm.
3700 if (!SrcVT.isVector() || !DstVT.isVector() ||
3701 SrcVT.getVectorElementType() == MVT::i1 ||
3702 DstVT.getVectorElementType() == MVT::i1)
3703 return false;
3704
3705 Register Reg = getRegForValue(I->getOperand(0));
3706 if (!Reg)
3707 return false;
3708
3709 // Emit a reg-reg copy so we don't propagate cached known bits information
3710 // with the wrong VT if we fall out of fast isel after selecting this.
3711 const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
3712 Register ResultReg = createResultReg(DstClass);
3713 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3714 TII.get(TargetOpcode::COPY), ResultReg).addReg(Reg);
3715
3716 updateValueMap(I, ResultReg);
3717 return true;
3718 }
3719 }
3720
3721 return false;
3722}
3723
3724unsigned X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
3725 if (VT > MVT::i64)
3726 return 0;
3727
3728 uint64_t Imm = CI->getZExtValue();
3729 if (Imm == 0) {
3730 Register SrcReg = fastEmitInst_(X86::MOV32r0, &X86::GR32RegClass);
3731 switch (VT.SimpleTy) {
3732 default: llvm_unreachable("Unexpected value type");
3733 case MVT::i1:
3734 case MVT::i8:
3735 return fastEmitInst_extractsubreg(MVT::i8, SrcReg, X86::sub_8bit);
3736 case MVT::i16:
3737 return fastEmitInst_extractsubreg(MVT::i16, SrcReg, X86::sub_16bit);
3738 case MVT::i32:
3739 return SrcReg;
3740 case MVT::i64: {
3741 Register ResultReg = createResultReg(&X86::GR64RegClass);
3742 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3743 TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)
3744 .addImm(0).addReg(SrcReg).addImm(X86::sub_32bit);
3745 return ResultReg;
3746 }
3747 }
3748 }
3749
3750 unsigned Opc = 0;
3751 switch (VT.SimpleTy) {
3752 default: llvm_unreachable("Unexpected value type");
3753 case MVT::i1:
3754 VT = MVT::i8;
3755 [[fallthrough]];
3756 case MVT::i8: Opc = X86::MOV8ri; break;
3757 case MVT::i16: Opc = X86::MOV16ri; break;
3758 case MVT::i32: Opc = X86::MOV32ri; break;
3759 case MVT::i64: {
3760 if (isUInt<32>(Imm))
3761 Opc = X86::MOV32ri64;
3762 else if (isInt<32>(Imm))
3763 Opc = X86::MOV64ri32;
3764 else
3765 Opc = X86::MOV64ri;
3766 break;
3767 }
3768 }
3769 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
3770}
3771
3772unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
3773 if (CFP->isNullValue())
3774 return fastMaterializeFloatZero(CFP);
3775
3776 // Can't handle alternate code models yet.
3777 CodeModel::Model CM = TM.getCodeModel();
3778 if (CM != CodeModel::Small && CM != CodeModel::Medium &&
3779 CM != CodeModel::Large)
3780 return 0;
3781
3782 // Get opcode and regclass of the output for the given load instruction.
3783 unsigned Opc = 0;
3784 bool HasSSE1 = Subtarget->hasSSE1();
3785 bool HasSSE2 = Subtarget->hasSSE2();
3786 bool HasAVX = Subtarget->hasAVX();
3787 bool HasAVX512 = Subtarget->hasAVX512();
3788 switch (VT.SimpleTy) {
3789 default: return 0;
3790 case MVT::f32:
3791 Opc = HasAVX512 ? X86::VMOVSSZrm_alt
3792 : HasAVX ? X86::VMOVSSrm_alt
3793 : HasSSE1 ? X86::MOVSSrm_alt
3794 : X86::LD_Fp32m;
3795 break;
3796 case MVT::f64:
3797 Opc = HasAVX512 ? X86::VMOVSDZrm_alt
3798 : HasAVX ? X86::VMOVSDrm_alt
3799 : HasSSE2 ? X86::MOVSDrm_alt
3800 : X86::LD_Fp64m;
3801 break;
3802 case MVT::f80:
3803 // No f80 support yet.
3804 return 0;
3805 }
3806
3807 // MachineConstantPool wants an explicit alignment.
3808 Align Alignment = DL.getPrefTypeAlign(CFP->getType());
3809
3810 // x86-32 PIC requires a PIC base register for constant pools.
3811 unsigned PICBase = 0;
3812 unsigned char OpFlag = Subtarget->classifyLocalReference(nullptr);
3813 if (OpFlag == X86II::MO_PIC_BASE_OFFSET)
3814 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3815 else if (OpFlag == X86II::MO_GOTOFF)
3816 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3817 else if (Subtarget->is64Bit() && TM.getCodeModel() != CodeModel::Large)
3818 PICBase = X86::RIP;
3819
3820 // Create the load from the constant pool.
3821 unsigned CPI = MCP.getConstantPoolIndex(CFP, Alignment);
3822 Register ResultReg = createResultReg(TLI.getRegClassFor(VT.SimpleTy));
3823
3824 // Large code model only applies to 64-bit mode.
3825 if (Subtarget->is64Bit() && CM == CodeModel::Large) {
3826 Register AddrReg = createResultReg(&X86::GR64RegClass);
3827 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV64ri),
3828 AddrReg)
3829 .addConstantPoolIndex(CPI, 0, OpFlag);
3830 MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3831 TII.get(Opc), ResultReg);
3832 addRegReg(MIB, AddrReg, false, PICBase, false);
3833 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3835 MachineMemOperand::MOLoad, DL.getPointerSize(), Alignment);
3836 MIB->addMemOperand(*FuncInfo.MF, MMO);
3837 return ResultReg;
3838 }
3839
3840 addConstantPoolReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3841 TII.get(Opc), ResultReg),
3842 CPI, PICBase, OpFlag);
3843 return ResultReg;
3844}
3845
3846unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
3847 // Can't handle large GlobalValues yet.
3848 if (TM.getCodeModel() != CodeModel::Small &&
3849 TM.getCodeModel() != CodeModel::Medium)
3850 return 0;
3851 if (TM.isLargeGlobalValue(GV))
3852 return 0;
3853
3854 // Materialize addresses with LEA/MOV instructions.
3855 X86AddressMode AM;
3856 if (X86SelectAddress(GV, AM)) {
3857 // If the expression is just a basereg, then we're done, otherwise we need
3858 // to emit an LEA.
3860 AM.IndexReg == 0 && AM.Disp == 0 && AM.GV == nullptr)
3861 return AM.Base.Reg;
3862
3863 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3864 if (TM.getRelocationModel() == Reloc::Static &&
3865 TLI.getPointerTy(DL) == MVT::i64) {
3866 // The displacement code could be more than 32 bits away so we need to use
3867 // an instruction with a 64 bit immediate
3868 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV64ri),
3869 ResultReg)
3870 .addGlobalAddress(GV);
3871 } else {
3872 unsigned Opc =
3873 TLI.getPointerTy(DL) == MVT::i32
3874 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3875 : X86::LEA64r;
3876 addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3877 TII.get(Opc), ResultReg), AM);
3878 }
3879 return ResultReg;
3880 }
3881 return 0;
3882}
3883
3884unsigned X86FastISel::fastMaterializeConstant(const Constant *C) {
3885 EVT CEVT = TLI.getValueType(DL, C->getType(), true);
3886
3887 // Only handle simple types.
3888 if (!CEVT.isSimple())
3889 return 0;
3890 MVT VT = CEVT.getSimpleVT();
3891
3892 if (const auto *CI = dyn_cast<ConstantInt>(C))
3893 return X86MaterializeInt(CI, VT);
3894 if (const auto *CFP = dyn_cast<ConstantFP>(C))
3895 return X86MaterializeFP(CFP, VT);
3896 if (const auto *GV = dyn_cast<GlobalValue>(C))
3897 return X86MaterializeGV(GV, VT);
3898 if (isa<UndefValue>(C)) {
3899 unsigned Opc = 0;
3900 switch (VT.SimpleTy) {
3901 default:
3902 break;
3903 case MVT::f32:
3904 if (!Subtarget->hasSSE1())
3905 Opc = X86::LD_Fp032;
3906 break;
3907 case MVT::f64:
3908 if (!Subtarget->hasSSE2())
3909 Opc = X86::LD_Fp064;
3910 break;
3911 case MVT::f80:
3912 Opc = X86::LD_Fp080;
3913 break;
3914 }
3915
3916 if (Opc) {
3917 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3918 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc),
3919 ResultReg);
3920 return ResultReg;
3921 }
3922 }
3923
3924 return 0;
3925}
3926
3927unsigned X86FastISel::fastMaterializeAlloca(const AllocaInst *C) {
3928 // Fail on dynamic allocas. At this point, getRegForValue has already
3929 // checked its CSE maps, so if we're here trying to handle a dynamic
3930 // alloca, we're not going to succeed. X86SelectAddress has a
3931 // check for dynamic allocas, because it's called directly from
3932 // various places, but targetMaterializeAlloca also needs a check
3933 // in order to avoid recursion between getRegForValue,
3934 // X86SelectAddrss, and targetMaterializeAlloca.
3935 if (!FuncInfo.StaticAllocaMap.count(C))
3936 return 0;
3937 assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?");
3938
3939 X86AddressMode AM;
3940 if (!X86SelectAddress(C, AM))
3941 return 0;
3942 unsigned Opc =
3943 TLI.getPointerTy(DL) == MVT::i32
3944 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3945 : X86::LEA64r;
3946 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
3947 Register ResultReg = createResultReg(RC);
3948 addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3949 TII.get(Opc), ResultReg), AM);
3950 return ResultReg;
3951}
3952
3953unsigned X86FastISel::fastMaterializeFloatZero(const ConstantFP *CF) {
3954 MVT VT;
3955 if (!isTypeLegal(CF->getType(), VT))
3956 return 0;
3957
3958 // Get opcode and regclass for the given zero.
3959 bool HasSSE1 = Subtarget->hasSSE1();
3960 bool HasSSE2 = Subtarget->hasSSE2();
3961 bool HasAVX512 = Subtarget->hasAVX512();
3962 unsigned Opc = 0;
3963 switch (VT.SimpleTy) {
3964 default: return 0;
3965 case MVT::f16:
3966 Opc = HasAVX512 ? X86::AVX512_FsFLD0SH : X86::FsFLD0SH;
3967 break;
3968 case MVT::f32:
3969 Opc = HasAVX512 ? X86::AVX512_FsFLD0SS
3970 : HasSSE1 ? X86::FsFLD0SS
3971 : X86::LD_Fp032;
3972 break;
3973 case MVT::f64:
3974 Opc = HasAVX512 ? X86::AVX512_FsFLD0SD
3975 : HasSSE2 ? X86::FsFLD0SD
3976 : X86::LD_Fp064;
3977 break;
3978 case MVT::f80:
3979 // No f80 support yet.
3980 return 0;
3981 }
3982
3983 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3984 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg);
3985 return ResultReg;
3986}
3987
3988
3989bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
3990 const LoadInst *LI) {
3991 const Value *Ptr = LI->getPointerOperand();
3992 X86AddressMode AM;
3993 if (!X86SelectAddress(Ptr, AM))
3994 return false;
3995
3996 const X86InstrInfo &XII = (const X86InstrInfo &)TII;
3997
3998 unsigned Size = DL.getTypeAllocSize(LI->getType());
3999
4001 AM.getFullAddress(AddrOps);
4002
4004 *FuncInfo.MF, *MI, OpNo, AddrOps, FuncInfo.InsertPt, Size, LI->getAlign(),
4005 /*AllowCommute=*/true);
4006 if (!Result)
4007 return false;
4008
4009 // The index register could be in the wrong register class. Unfortunately,
4010 // foldMemoryOperandImpl could have commuted the instruction so its not enough
4011 // to just look at OpNo + the offset to the index reg. We actually need to
4012 // scan the instruction to find the index reg and see if its the correct reg
4013 // class.
4014 unsigned OperandNo = 0;
4015 for (MachineInstr::mop_iterator I = Result->operands_begin(),
4016 E = Result->operands_end(); I != E; ++I, ++OperandNo) {
4017 MachineOperand &MO = *I;
4018 if (!MO.isReg() || MO.isDef() || MO.getReg() != AM.IndexReg)
4019 continue;
4020 // Found the index reg, now try to rewrite it.
4021 Register IndexReg = constrainOperandRegClass(Result->getDesc(),
4022 MO.getReg(), OperandNo);
4023 if (IndexReg == MO.getReg())
4024 continue;
4025 MO.setReg(IndexReg);
4026 }
4027
4028 Result->addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI));
4029 Result->cloneInstrSymbols(*FuncInfo.MF, *MI);
4031 removeDeadCode(I, std::next(I));
4032 return true;
4033}
4034
4035unsigned X86FastISel::fastEmitInst_rrrr(unsigned MachineInstOpcode,
4036 const TargetRegisterClass *RC,
4037 unsigned Op0, unsigned Op1,
4038 unsigned Op2, unsigned Op3) {
4039 const MCInstrDesc &II = TII.get(MachineInstOpcode);
4040
4041 Register ResultReg = createResultReg(RC);
4042 Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
4043 Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
4044 Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
4045 Op3 = constrainOperandRegClass(II, Op3, II.getNumDefs() + 3);
4046
4047 if (II.getNumDefs() >= 1)
4048 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
4049 .addReg(Op0)
4050 .addReg(Op1)
4051 .addReg(Op2)
4052 .addReg(Op3);
4053 else {
4054 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
4055 .addReg(Op0)
4056 .addReg(Op1)
4057 .addReg(Op2)
4058 .addReg(Op3);
4059 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
4060 ResultReg)
4061 .addReg(II.implicit_defs()[0]);
4062 }
4063 return ResultReg;
4064}
4065
4066
4067namespace llvm {
4069 const TargetLibraryInfo *libInfo) {
4070 return new X86FastISel(funcInfo, libInfo);
4071 }
4072}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
This file defines the FastISel class.
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned OpIndex
static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC)
If we have a comparison with RHS as the RHS of the comparison, return an opcode that works for the co...
#define GET_EGPR_IF_ENABLED(OPC)
static std::pair< unsigned, bool > getX86SSEConditionCode(CmpInst::Predicate Predicate)
static unsigned computeBytesPoppedByCalleeForSRet(const X86Subtarget *Subtarget, CallingConv::ID CC, const CallBase *CB)
static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget)
static void X86SelectAddress(const MachineInstr &I, const MachineRegisterInfo &MRI, X86AddressMode &AM)
Value * RHS
Value * LHS
an instruction to allocate memory on the stack
Definition: Instructions.h:59
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
InstListType::const_iterator const_iterator
Definition: BasicBlock.h:165
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
Value * getCondition() const
CCState - This class holds information needed while lowering arguments and return values.
CCValAssign - Represent assignment of one arg/retval to a location.
bool isRegLoc() const
Register getLocReg() const
LocInfo getLocInfo() const
bool isMemLoc() const
int64_t getLocMemOffset() const
unsigned getValNo() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1455
bool arg_empty() const
Definition: InstrTypes.h:1645
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1703
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1648
unsigned arg_size() const
Definition: InstrTypes.h:1646
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:955
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:968
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:982
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:971
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:980
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:969
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:970
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:979
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:973
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:976
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:977
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:972
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:974
@ ICMP_NE
not equal
Definition: InstrTypes.h:987
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:981
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:978
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:967
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:975
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:1090
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:1066
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1016
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:267
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:159
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:90
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
This class represents an Operation in the Expression.
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
A debug info location.
Definition: DebugLoc.h:33
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
virtual unsigned fastMaterializeFloatZero(const ConstantFP *CF)
Emit the floating-point constant +0.0 in a register using target- specific logic.
Definition: FastISel.h:480
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering.
Definition: FastISel.cpp:1930
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:300
virtual unsigned fastMaterializeConstant(const Constant *C)
Emit a constant in a register using target-specific logic, such as constant pool loads.
Definition: FastISel.h:473
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1928
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1926
virtual bool fastSelectInstruction(const Instruction *I)=0
This method is called by target-independent code when the normal FastISel process fails to select an ...
const TargetMachine & TM
Definition: FastISel.h:209
virtual unsigned fastMaterializeAlloca(const AllocaInst *C)
Emit an alloca address in a register using target-specific logic.
Definition: FastISel.h:476
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:205
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:973
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:453
bool isAtomic() const LLVM_READONLY
Return true if this instruction has an AtomicOrdering of unordered or higher.
const BasicBlock * getParent() const
Definition: Instruction.h:151
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
An instruction for reading from memory.
Definition: Instructions.h:184
Value * getPointerOperand()
Definition: Instructions.h:280
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:236
bool usesWindowsCFI() const
Definition: MCAsmInfo.h:799
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:248
ArrayRef< MCPhysReg > implicit_defs() const
Return a list of registers that are potentially written by any instance of this machine instruction.
Definition: MCInstrDesc.h:579
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Machine Value Type.
SimpleValueType SimpleTy
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
MVT getVectorElementType() const
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setStackProtectorIndex(int I)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:547
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:541
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
This class wraps the llvm.memcpy intrinsic.
Value * getLength() const
Value * getRawDest() const
unsigned getDestAddressSpace() const
bool isVolatile() const
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
Value * getRawSource() const
Return the arguments to the instruction.
unsigned getSourceAddressSpace() const
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Return a value (possibly void), from a function.
size_t size() const
Definition: SmallVector.h:91
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:317
Align getAlign() const
Definition: Instructions.h:369
Value * getValueOperand()
Definition: Instructions.h:414
Value * getPointerOperand()
Definition: Instructions.h:417
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:622
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
Provides information about what library functions are available for the current target.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
Definition: Triple.h:651
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
bool isArrayTy() const
True if this is an instance of ArrayType.
Definition: Type.h:252
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition: Type.h:154
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:249
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
Fold a load or store of the specified stack slot into the specified machine instruction for the speci...
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
bool hasSSE1() const
Definition: X86Subtarget.h:200
bool isTargetMCU() const
Definition: X86Subtarget.h:310
const Triple & getTargetTriple() const
Definition: X86Subtarget.h:291
bool hasAVX512() const
Definition: X86Subtarget.h:208
bool hasSSE2() const
Definition: X86Subtarget.h:201
bool hasAVX() const
Definition: X86Subtarget.h:206
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ X86_64_SysV
The C convention as specified in the x86-64 supplement to the System V ABI, used on most non-Windows ...
Definition: CallingConv.h:151
@ HiPE
Used by the High-Performance Erlang Compiler (HiPE).
Definition: CallingConv.h:53
@ Swift
Calling convention for Swift.
Definition: CallingConv.h:69
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
Definition: CallingConv.h:82
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
Definition: CallingConv.h:50
@ X86_ThisCall
Similar to X86_StdCall.
Definition: CallingConv.h:122
@ X86_StdCall
stdcall is mostly used by the Win32 API.
Definition: CallingConv.h:99
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition: CallingConv.h:76
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
Definition: CallingConv.h:159
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
Definition: CallingConv.h:87
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ X86_FastCall
'fast' analog of X86_StdCall.
Definition: CallingConv.h:103
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:620
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
@ MO_GOTPCREL_NORELAX
MO_GOTPCREL_NORELAX - Same as MO_GOTPCREL except that R_X86_64_GOTPCREL relocations are guaranteed to...
Definition: X86BaseInfo.h:405
@ MO_GOTOFF
MO_GOTOFF - On a symbol operand this indicates that the immediate is the offset to the location of th...
Definition: X86BaseInfo.h:395
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
Definition: X86BaseInfo.h:502
@ MO_PLT
MO_PLT - On a symbol operand this indicates that the immediate is offset to the PLT entry of symbol n...
Definition: X86BaseInfo.h:410
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
Definition: X86BaseInfo.h:377
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand "FOO", this indicates that the reference is actually to the "__imp...
Definition: X86BaseInfo.h:474
@ MO_PIC_BASE_OFFSET
MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the immediate should get the value of th...
Definition: X86BaseInfo.h:385
@ MO_GOTPCREL
MO_GOTPCREL - On a symbol operand this indicates that the immediate is offset to the GOT entry for th...
Definition: X86BaseInfo.h:401
@ AddrIndexReg
Definition: X86BaseInfo.h:31
@ LAST_VALID_COND
Definition: X86BaseInfo.h:92
std::pair< CondCode, bool > getX86ConditionCode(CmpInst::Predicate Predicate)
Return a pair of condition code for the given predicate and whether the instruction operands should b...
bool isCalleePop(CallingConv::ID CallingConv, bool is64Bit, bool IsVarArg, bool GuaranteeTCO)
Determines whether the callee is required to pop its own arguments.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand=false, bool HasNDD=false)
Return a cmov opcode for the given register size in bytes, and operand type.
StringMapEntry< std::atomic< TypeEntryBody * > > TypeEntry
Definition: TypePool.h:27
constexpr double e
Definition: MathExtras.h:31
@ Emitted
Assigned address, still materializing.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
static bool isGlobalStubReference(unsigned char TargetFlag)
isGlobalStubReference - Return true if the specified TargetFlag operand is a reference to a stub for ...
Definition: X86InstrInfo.h:103
static bool isGlobalRelativeToPICBase(unsigned char TargetFlag)
isGlobalRelativeToPICBase - Return true if the specified global value reference is relative to a 32-b...
Definition: X86InstrInfo.h:121
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
Definition: Utils.cpp:54
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto successors(const MachineBasicBlock *BB)
bool RetCC_X86(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)
addFrameReference - This function is used to add a reference to the base of an abstract object on the...
static const MachineInstrBuilder & addFullAddress(const MachineInstrBuilder &MIB, const X86AddressMode &AM)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
static const MachineInstrBuilder & addConstantPoolReference(const MachineInstrBuilder &MIB, unsigned CPI, unsigned GlobalBaseReg, unsigned char OpFlags)
addConstantPoolReference - This function is used to add a reference to the base of a constant value s...
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
static const MachineInstrBuilder & addRegReg(const MachineInstrBuilder &MIB, unsigned Reg1, bool isKill1, unsigned Reg2, bool isKill2)
addRegReg - This function is used to add a memory reference of the form: [Reg + Reg].
unsigned getKillRegState(bool B)
gep_type_iterator gep_type_begin(const User *GEP)
bool CC_X86(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static const MachineInstrBuilder & addDirectMem(const MachineInstrBuilder &MIB, unsigned Reg)
addDirectMem - This function is used to add a direct memory reference to the current instruction – th...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Description of the encoding of one expression Op.
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
X86AddressMode - This struct holds a generalized full x86 address mode.
enum llvm::X86AddressMode::@614 BaseType
void getFullAddress(SmallVectorImpl< MachineOperand > &MO)
const GlobalValue * GV
union llvm::X86AddressMode::@615 Base