LLVM 24.0.0git
R600ISelLowering.cpp
Go to the documentation of this file.
1//===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for R600
11//
12//===----------------------------------------------------------------------===//
13
14#include "R600ISelLowering.h"
15#include "AMDGPU.h"
18#include "R600Defines.h"
20#include "R600Subtarget.h"
22#include "llvm/IR/IntrinsicsAMDGPU.h"
23#include "llvm/IR/IntrinsicsR600.h"
24
25using namespace llvm;
26
27#define GET_CALLING_CONV_IMPL
28#include "R600GenCallingConv.inc"
29
31 const R600Subtarget &STI)
32 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI),
33 Gen(STI.getGeneration()) {
34 addRegisterClass(MVT::f32, &R600::R600_Reg32RegClass);
35 addRegisterClass(MVT::i32, &R600::R600_Reg32RegClass);
36 addRegisterClass(MVT::v2f32, &R600::R600_Reg64RegClass);
37 addRegisterClass(MVT::v2i32, &R600::R600_Reg64RegClass);
38 addRegisterClass(MVT::v4f32, &R600::R600_Reg128RegClass);
39 addRegisterClass(MVT::v4i32, &R600::R600_Reg128RegClass);
40
43
44 computeRegisterProperties(Subtarget->getRegisterInfo());
45
46 // Legalize loads and stores to the private address space.
47 setOperationAction(ISD::LOAD, {MVT::i32, MVT::v2i32, MVT::v4i32}, Custom);
48
49 // EXTLOAD should be the same as ZEXTLOAD. It is legal for some address
50 // spaces, so it is custom lowered to handle those where it isn't.
52 for (MVT VT : MVT::integer_valuetypes()) {
53 setLoadExtAction(Op, VT, MVT::i1, Promote);
54 setLoadExtAction(Op, VT, MVT::i8, Custom);
55 setLoadExtAction(Op, VT, MVT::i16, Custom);
56 }
57
58 // Workaround for LegalizeDAG asserting on expansion of i1 vector loads.
60 MVT::v2i1, Expand);
61
63 MVT::v4i1, Expand);
64
65 setOperationAction(ISD::STORE, {MVT::i8, MVT::i32, MVT::v2i32, MVT::v4i32},
66 Custom);
67
68 setTruncStoreAction(MVT::i32, MVT::i8, Custom);
69 setTruncStoreAction(MVT::i32, MVT::i16, Custom);
70 // We need to include these since trunc STORES to PRIVATE need
71 // special handling to accommodate RMW
72 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom);
73 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Custom);
74 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Custom);
75 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Custom);
76 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Custom);
77 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom);
78 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom);
79 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Custom);
80 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Custom);
81 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Custom);
82
83 // Workaround for LegalizeDAG asserting on expansion of i1 vector stores.
84 setTruncStoreAction(MVT::v2i32, MVT::v2i1, Expand);
85 setTruncStoreAction(MVT::v4i32, MVT::v4i1, Expand);
86
87 // Set condition code actions
91 MVT::f32, Expand);
92
94 MVT::i32, Expand);
95
97
98 setOperationAction(ISD::SETCC, {MVT::v4i32, MVT::v2i32}, Expand);
99
100 setOperationAction(ISD::BR_CC, {MVT::i32, MVT::f32}, Expand);
102
104
106 {MVT::f32, MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
107 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32},
108 Expand);
109
111 MVT::f64, Custom);
112
114
115 setOperationAction(ISD::SELECT_CC, {MVT::f32, MVT::i32}, Custom);
116
117 setOperationAction(ISD::SETCC, {MVT::i32, MVT::f32}, Expand);
118 setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT}, {MVT::i1, MVT::i64},
119 Custom);
120
121 setOperationAction(ISD::SELECT, {MVT::i32, MVT::f32, MVT::v2i32, MVT::v4i32},
122 Expand);
123
124 // ADD, SUB overflow.
125 // TODO: turn these into Legal?
126 if (Subtarget->hasCARRY())
128
129 if (Subtarget->hasBORROW())
131
132 // Expand sign extension of vectors
133 if (!Subtarget->hasBFE())
135
136 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::v2i1, MVT::v4i1}, Expand);
137
138 if (!Subtarget->hasBFE())
140 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::v2i8, MVT::v4i8}, Expand);
141
142 if (!Subtarget->hasBFE())
144 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::v2i16, MVT::v4i16}, Expand);
145
147 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::v2i32, MVT::v4i32}, Expand);
148
150
152
154 {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Custom);
155
157 {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Custom);
158
159 // We don't have 64-bit shifts. Thus we need either SHX i64 or SHX_PARTS i32
160 // to be Legal/Custom in order to avoid library calls.
162 Custom);
163
164 if (!Subtarget->hasFMA())
165 setOperationAction(ISD::FMA, {MVT::f32, MVT::f64}, Expand);
166
167 // FIXME: May need no denormals check
169
170 if (!Subtarget->hasBFI())
171 // fcopysign can be done in a single instruction with BFI.
172 setOperationAction(ISD::FCOPYSIGN, {MVT::f32, MVT::f64}, Expand);
173
174 if (!Subtarget->hasBCNT(32))
176
177 if (!Subtarget->hasBCNT(64))
179
180 if (Subtarget->hasFFBH())
182
183 if (Subtarget->hasFFBL())
185
186 // FIXME: This was moved from AMDGPUTargetLowering, I'm not sure if we
187 // need it for R600.
188 if (Subtarget->hasBFE())
190
193
194 // LLVM will expand these to atomic_cmp_swap(0)
195 // and atomic_swap, respectively.
197
198 // We need to custom lower some of the intrinsics
200 Custom);
201
203
206}
207
209 if (std::next(I) == I->getParent()->end())
210 return false;
211 return std::next(I)->getOpcode() == R600::RETURN;
212}
213
216 MachineBasicBlock *BB) const {
217 MachineFunction *MF = BB->getParent();
220 const R600InstrInfo *TII = Subtarget->getInstrInfo();
221
222 switch (MI.getOpcode()) {
223 default:
224 // Replace LDS_*_RET instruction that don't have any uses with the
225 // equivalent LDS_*_NORET instruction.
226 if (TII->isLDSRetInstr(MI.getOpcode())) {
227 int DstIdx = TII->getOperandIdx(MI.getOpcode(), R600::OpName::dst);
228 assert(DstIdx != -1);
230 // FIXME: getLDSNoRetOp method only handles LDS_1A1D LDS ops. Add
231 // LDS_1A2D support and remove this special case.
232 if (!MRI.use_empty(MI.getOperand(DstIdx).getReg()) ||
233 MI.getOpcode() == R600::LDS_CMPST_RET)
234 return BB;
235
236 NewMI = BuildMI(*BB, I, BB->findDebugLoc(I),
237 TII->get(R600::getLDSNoRetOp(MI.getOpcode())));
238 for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))
239 NewMI.add(MO);
240 } else {
242 }
243 break;
244
245 case R600::FABS_R600: {
246 MachineInstr *NewMI = TII->buildDefaultInstruction(
247 *BB, I, R600::MOV, MI.getOperand(0).getReg(),
248 MI.getOperand(1).getReg());
249 TII->addFlag(*NewMI, 0, MO_FLAG_ABS);
250 break;
251 }
252
253 case R600::FNEG_R600: {
254 MachineInstr *NewMI = TII->buildDefaultInstruction(
255 *BB, I, R600::MOV, MI.getOperand(0).getReg(),
256 MI.getOperand(1).getReg());
257 TII->addFlag(*NewMI, 0, MO_FLAG_NEG);
258 break;
259 }
260
261 case R600::MASK_WRITE: {
262 Register maskedRegister = MI.getOperand(0).getReg();
263 assert(maskedRegister.isVirtual());
264 MachineInstr * defInstr = MRI.getVRegDef(maskedRegister);
265 TII->addFlag(*defInstr, 0, MO_FLAG_MASK);
266 break;
267 }
268
269 case R600::MOV_IMM_F32:
270 TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(), MI.getOperand(1)
271 .getFPImm()
272 ->getValueAPF()
273 .bitcastToAPInt()
274 .getZExtValue());
275 break;
276
277 case R600::MOV_IMM_I32:
278 TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(),
279 MI.getOperand(1).getImm());
280 break;
281
282 case R600::MOV_IMM_GLOBAL_ADDR: {
283 //TODO: Perhaps combine this instruction with the next if possible
284 auto MIB = TII->buildDefaultInstruction(
285 *BB, MI, R600::MOV, MI.getOperand(0).getReg(), R600::ALU_LITERAL_X);
286 int Idx = TII->getOperandIdx(*MIB, R600::OpName::literal);
287 //TODO: Ugh this is rather ugly
288 const MachineOperand &MO = MI.getOperand(1);
289 MIB->getOperand(Idx).ChangeToGA(MO.getGlobal(), MO.getOffset(),
290 MO.getTargetFlags());
291 break;
292 }
293
294 case R600::CONST_COPY: {
295 MachineInstr *NewMI = TII->buildDefaultInstruction(
296 *BB, MI, R600::MOV, MI.getOperand(0).getReg(), R600::ALU_CONST);
297 TII->setImmOperand(*NewMI, R600::OpName::src0_sel,
298 MI.getOperand(1).getImm());
299 break;
300 }
301
302 case R600::RAT_WRITE_CACHELESS_32_eg:
303 case R600::RAT_WRITE_CACHELESS_64_eg:
304 case R600::RAT_WRITE_CACHELESS_128_eg:
305 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode()))
306 .add(MI.getOperand(0))
307 .add(MI.getOperand(1))
308 .addImm(isEOP(I)); // Set End of program bit
309 break;
310
311 case R600::RAT_STORE_TYPED_eg:
312 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode()))
313 .add(MI.getOperand(0))
314 .add(MI.getOperand(1))
315 .add(MI.getOperand(2))
316 .addImm(isEOP(I)); // Set End of program bit
317 break;
318
319 case R600::BRANCH:
320 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP))
321 .add(MI.getOperand(0));
322 break;
323
324 case R600::BRANCH_COND_f32: {
325 MachineInstr *NewMI =
326 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::PRED_X),
327 R600::PREDICATE_BIT)
328 .add(MI.getOperand(1))
329 .addImm(R600::PRED_SETNE)
330 .addImm(0); // Flags
331 TII->addFlag(*NewMI, 0, MO_FLAG_PUSH);
332 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP_COND))
333 .add(MI.getOperand(0))
334 .addReg(R600::PREDICATE_BIT, RegState::Kill);
335 break;
336 }
337
338 case R600::BRANCH_COND_i32: {
339 MachineInstr *NewMI =
340 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::PRED_X),
341 R600::PREDICATE_BIT)
342 .add(MI.getOperand(1))
343 .addImm(R600::PRED_SETNE_INT)
344 .addImm(0); // Flags
345 TII->addFlag(*NewMI, 0, MO_FLAG_PUSH);
346 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(R600::JUMP_COND))
347 .add(MI.getOperand(0))
348 .addReg(R600::PREDICATE_BIT, RegState::Kill);
349 break;
350 }
351
352 case R600::EG_ExportSwz:
353 case R600::R600_ExportSwz: {
354 // Instruction is left unmodified if its not the last one of its type
355 bool isLastInstructionOfItsType = true;
356 unsigned InstExportType = MI.getOperand(1).getImm();
357 for (MachineBasicBlock::iterator NextExportInst = std::next(I),
358 EndBlock = BB->end(); NextExportInst != EndBlock;
359 NextExportInst = std::next(NextExportInst)) {
360 if (NextExportInst->getOpcode() == R600::EG_ExportSwz ||
361 NextExportInst->getOpcode() == R600::R600_ExportSwz) {
362 unsigned CurrentInstExportType = NextExportInst->getOperand(1)
363 .getImm();
364 if (CurrentInstExportType == InstExportType) {
365 isLastInstructionOfItsType = false;
366 break;
367 }
368 }
369 }
370 bool EOP = isEOP(I);
371 if (!EOP && !isLastInstructionOfItsType)
372 return BB;
373 unsigned CfInst = (MI.getOpcode() == R600::EG_ExportSwz) ? 84 : 40;
374 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI.getOpcode()))
375 .add(MI.getOperand(0))
376 .add(MI.getOperand(1))
377 .add(MI.getOperand(2))
378 .add(MI.getOperand(3))
379 .add(MI.getOperand(4))
380 .add(MI.getOperand(5))
381 .add(MI.getOperand(6))
382 .addImm(CfInst)
383 .addImm(EOP);
384 break;
385 }
386 case R600::RETURN: {
387 return BB;
388 }
389 }
390
391 MI.eraseFromParent();
392 return BB;
393}
394
395//===----------------------------------------------------------------------===//
396// Custom DAG Lowering Operations
397//===----------------------------------------------------------------------===//
398
402 switch (Op.getOpcode()) {
403 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
404 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
405 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
406 case ISD::SHL_PARTS:
407 case ISD::SRA_PARTS:
408 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
409 case ISD::UADDO: return LowerUADDSUBO(Op, DAG, ISD::ADD, AMDGPUISD::CARRY);
410 case ISD::USUBO: return LowerUADDSUBO(Op, DAG, ISD::SUB, AMDGPUISD::BORROW);
411 case ISD::FCOS:
412 case ISD::FSIN: return LowerTrig(Op, DAG);
413 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
414 case ISD::STORE: return LowerSTORE(Op, DAG);
415 case ISD::LOAD: {
416 SDValue Result = LowerLOAD(Op, DAG);
417 assert((!Result.getNode() ||
418 Result.getNode()->getNumValues() == 2) &&
419 "Load should return a value and a chain");
420 return Result;
421 }
422
423 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
424 case ISD::GlobalAddress: return LowerGlobalAddress(MFI, Op, DAG);
425 case ISD::FrameIndex: return lowerFrameIndex(Op, DAG);
427 return lowerADDRSPACECAST(Op, DAG);
428 case ISD::INTRINSIC_VOID: {
429 SDValue Chain = Op.getOperand(0);
430 unsigned IntrinsicID = Op.getConstantOperandVal(1);
431 switch (IntrinsicID) {
432 case Intrinsic::r600_store_swizzle: {
433 SDLoc DL(Op);
434 const SDValue Args[8] = {
435 Chain,
436 Op.getOperand(2), // Export Value
437 Op.getOperand(3), // ArrayBase
438 Op.getOperand(4), // Type
439 DAG.getConstant(0, DL, MVT::i32), // SWZ_X
440 DAG.getConstant(1, DL, MVT::i32), // SWZ_Y
441 DAG.getConstant(2, DL, MVT::i32), // SWZ_Z
442 DAG.getConstant(3, DL, MVT::i32) // SWZ_W
443 };
444 return DAG.getNode(AMDGPUISD::R600_EXPORT, DL, Op.getValueType(), Args);
445 }
446
447 // default for switch(IntrinsicID)
448 default: break;
449 }
450 // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode())
451 break;
452 }
454 unsigned IntrinsicID = Op.getConstantOperandVal(0);
455 EVT VT = Op.getValueType();
456 SDLoc DL(Op);
457 switch (IntrinsicID) {
458 case Intrinsic::r600_tex:
459 case Intrinsic::r600_texc: {
460 unsigned TextureOp;
461 switch (IntrinsicID) {
462 case Intrinsic::r600_tex:
463 TextureOp = 0;
464 break;
465 case Intrinsic::r600_texc:
466 TextureOp = 1;
467 break;
468 default:
469 llvm_unreachable("unhandled texture operation");
470 }
471
472 SDValue TexArgs[19] = {
473 DAG.getConstant(TextureOp, DL, MVT::i32),
474 Op.getOperand(1),
475 DAG.getConstant(0, DL, MVT::i32),
476 DAG.getConstant(1, DL, MVT::i32),
477 DAG.getConstant(2, DL, MVT::i32),
478 DAG.getConstant(3, DL, MVT::i32),
479 Op.getOperand(2),
480 Op.getOperand(3),
481 Op.getOperand(4),
482 DAG.getConstant(0, DL, MVT::i32),
483 DAG.getConstant(1, DL, MVT::i32),
484 DAG.getConstant(2, DL, MVT::i32),
485 DAG.getConstant(3, DL, MVT::i32),
486 Op.getOperand(5),
487 Op.getOperand(6),
488 Op.getOperand(7),
489 Op.getOperand(8),
490 Op.getOperand(9),
491 Op.getOperand(10)
492 };
493 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, MVT::v4f32, TexArgs);
494 }
495 case Intrinsic::r600_dot4: {
496 SDValue Args[8] = {
497 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
498 DAG.getConstant(0, DL, MVT::i32)),
499 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
500 DAG.getConstant(0, DL, MVT::i32)),
501 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
502 DAG.getConstant(1, DL, MVT::i32)),
503 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
504 DAG.getConstant(1, DL, MVT::i32)),
505 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
506 DAG.getConstant(2, DL, MVT::i32)),
507 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
508 DAG.getConstant(2, DL, MVT::i32)),
509 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(1),
510 DAG.getConstant(3, DL, MVT::i32)),
511 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Op.getOperand(2),
512 DAG.getConstant(3, DL, MVT::i32))
513 };
514 return DAG.getNode(AMDGPUISD::DOT4, DL, MVT::f32, Args);
515 }
516
517 case Intrinsic::r600_implicitarg_ptr: {
520 return DAG.getConstant(ByteOffset, DL, PtrVT);
521 }
522 case Intrinsic::r600_read_ngroups_x:
523 return LowerImplicitParameter(DAG, VT, DL, 0);
524 case Intrinsic::r600_read_ngroups_y:
525 return LowerImplicitParameter(DAG, VT, DL, 1);
526 case Intrinsic::r600_read_ngroups_z:
527 return LowerImplicitParameter(DAG, VT, DL, 2);
528 case Intrinsic::r600_read_global_size_x:
529 return LowerImplicitParameter(DAG, VT, DL, 3);
530 case Intrinsic::r600_read_global_size_y:
531 return LowerImplicitParameter(DAG, VT, DL, 4);
532 case Intrinsic::r600_read_global_size_z:
533 return LowerImplicitParameter(DAG, VT, DL, 5);
534 case Intrinsic::r600_read_local_size_x:
535 return LowerImplicitParameter(DAG, VT, DL, 6);
536 case Intrinsic::r600_read_local_size_y:
537 return LowerImplicitParameter(DAG, VT, DL, 7);
538 case Intrinsic::r600_read_local_size_z:
539 return LowerImplicitParameter(DAG, VT, DL, 8);
540
541 case Intrinsic::r600_read_tgid_x:
542 case Intrinsic::amdgcn_workgroup_id_x:
543 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
544 R600::T1_X, VT);
545 case Intrinsic::r600_read_tgid_y:
546 case Intrinsic::amdgcn_workgroup_id_y:
547 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
548 R600::T1_Y, VT);
549 case Intrinsic::r600_read_tgid_z:
550 case Intrinsic::amdgcn_workgroup_id_z:
551 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
552 R600::T1_Z, VT);
553 case Intrinsic::r600_read_tidig_x:
554 case Intrinsic::amdgcn_workitem_id_x:
555 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
556 R600::T0_X, VT);
557 case Intrinsic::r600_read_tidig_y:
558 case Intrinsic::amdgcn_workitem_id_y:
559 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
560 R600::T0_Y, VT);
561 case Intrinsic::r600_read_tidig_z:
562 case Intrinsic::amdgcn_workitem_id_z:
563 return CreateLiveInRegisterRaw(DAG, &R600::R600_TReg32RegClass,
564 R600::T0_Z, VT);
565
566 case Intrinsic::r600_recipsqrt_ieee:
567 return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
568
569 case Intrinsic::r600_recipsqrt_clamped:
570 return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1));
571 default:
572 return Op;
573 }
574
575 // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode())
576 break;
577 }
578 } // end switch(Op.getOpcode())
579 return SDValue();
580}
581
584 SelectionDAG &DAG) const {
585 switch (N->getOpcode()) {
586 default:
588 return;
589 case ISD::FP_TO_UINT:
590 if (N->getValueType(0) == MVT::i1) {
591 Results.push_back(lowerFP_TO_UINT(N->getOperand(0), DAG));
592 return;
593 }
594 // Since we don't care about out of bounds values we can use FP_TO_SINT for
595 // uints too. The DAGLegalizer code for uint considers some extra cases
596 // which are not necessary here.
597 [[fallthrough]];
598 case ISD::FP_TO_SINT: {
599 if (N->getValueType(0) == MVT::i1) {
600 Results.push_back(lowerFP_TO_SINT(N->getOperand(0), DAG));
601 return;
602 }
603
604 SDValue Result;
605 if (expandFP_TO_SINT(N, Result, DAG))
606 Results.push_back(Result);
607 return;
608 }
609 case ISD::SDIVREM: {
610 SDValue Op = SDValue(N, 1);
611 SDValue RES = LowerSDIVREM(Op, DAG);
612 Results.push_back(RES);
613 Results.push_back(RES.getValue(1));
614 break;
615 }
616 case ISD::UDIVREM: {
617 SDValue Op = SDValue(N, 0);
619 break;
620 }
621 }
622}
623
624SDValue R600TargetLowering::vectorToVerticalVector(SelectionDAG &DAG,
625 SDValue Vector) const {
626 SDLoc DL(Vector);
627 EVT VecVT = Vector.getValueType();
628 EVT EltVT = VecVT.getVectorElementType();
630
631 for (unsigned i = 0, e = VecVT.getVectorNumElements(); i != e; ++i) {
632 Args.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vector,
633 DAG.getVectorIdxConstant(i, DL)));
634 }
635
636 return DAG.getNode(AMDGPUISD::BUILD_VERTICAL_VECTOR, DL, VecVT, Args);
637}
638
639SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
640 SelectionDAG &DAG) const {
641 SDLoc DL(Op);
642 SDValue Vector = Op.getOperand(0);
643 SDValue Index = Op.getOperand(1);
644
645 if (isa<ConstantSDNode>(Index) ||
647 return Op;
648
649 Vector = vectorToVerticalVector(DAG, Vector);
650 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(),
651 Vector, Index);
652}
653
654SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
655 SelectionDAG &DAG) const {
656 SDLoc DL(Op);
657 SDValue Vector = Op.getOperand(0);
658 SDValue Value = Op.getOperand(1);
659 SDValue Index = Op.getOperand(2);
660
661 if (isa<ConstantSDNode>(Index) ||
663 return Op;
664
665 Vector = vectorToVerticalVector(DAG, Vector);
666 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Op.getValueType(),
667 Vector, Value, Index);
668 return vectorToVerticalVector(DAG, Insert);
669}
670
671SDValue R600TargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
672 SDValue Op,
673 SelectionDAG &DAG) const {
674 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op);
677
678 const DataLayout &DL = DAG.getDataLayout();
679 const GlobalValue *GV = GSD->getGlobal();
680 MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
681
682 SDValue GA = DAG.getTargetGlobalAddress(GV, SDLoc(GSD), ConstPtrVT);
683 return DAG.getNode(AMDGPUISD::CONST_DATA_PTR, SDLoc(GSD), ConstPtrVT, GA);
684}
685
686SDValue R600TargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
687 // On hw >= R700, COS/SIN input must be between -1. and 1.
688 // Thus we lower them to TRIG ( FRACT ( x / 2Pi + 0.5) - 0.5)
689 EVT VT = Op.getValueType();
690 SDValue Arg = Op.getOperand(0);
691 SDLoc DL(Op);
692
693 // TODO: Should this propagate fast-math-flags?
694 SDValue FractPart = DAG.getNode(AMDGPUISD::FRACT, DL, VT,
695 DAG.getNode(ISD::FADD, DL, VT,
696 DAG.getNode(ISD::FMUL, DL, VT, Arg,
697 DAG.getConstantFP(0.15915494309, DL, MVT::f32)),
698 DAG.getConstantFP(0.5, DL, MVT::f32)));
699 unsigned TrigNode;
700 switch (Op.getOpcode()) {
701 case ISD::FCOS:
702 TrigNode = AMDGPUISD::COS_HW;
703 break;
704 case ISD::FSIN:
705 TrigNode = AMDGPUISD::SIN_HW;
706 break;
707 default:
708 llvm_unreachable("Wrong trig opcode");
709 }
710 SDValue TrigVal = DAG.getNode(TrigNode, DL, VT,
711 DAG.getNode(ISD::FADD, DL, VT, FractPart,
712 DAG.getConstantFP(-0.5, DL, MVT::f32)));
713 if (Gen >= AMDGPUSubtarget::R700)
714 return TrigVal;
715 // On R600 hw, COS/SIN input must be between -Pi and Pi.
716 return DAG.getNode(ISD::FMUL, DL, VT, TrigVal,
717 DAG.getConstantFP(numbers::pif, DL, MVT::f32));
718}
719
720SDValue R600TargetLowering::LowerShiftParts(SDValue Op,
721 SelectionDAG &DAG) const {
722 SDValue Lo, Hi;
723 expandShiftParts(Op.getNode(), Lo, Hi, DAG);
724 return DAG.getMergeValues({Lo, Hi}, SDLoc(Op));
725}
726
727SDValue R600TargetLowering::LowerUADDSUBO(SDValue Op, SelectionDAG &DAG,
728 unsigned mainop, unsigned ovf) const {
729 SDLoc DL(Op);
730 EVT VT = Op.getValueType();
731
732 SDValue Lo = Op.getOperand(0);
733 SDValue Hi = Op.getOperand(1);
734
735 SDValue OVF = DAG.getNode(ovf, DL, VT, Lo, Hi);
736 // Extend sign.
737 OVF = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, OVF,
738 DAG.getValueType(MVT::i1));
739
740 SDValue Res = DAG.getNode(mainop, DL, VT, Lo, Hi);
741
742 return DAG.getNode(ISD::MERGE_VALUES, DL, DAG.getVTList(VT, VT), Res, OVF);
743}
744
745SDValue R600TargetLowering::lowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const {
746 SDLoc DL(Op);
747 return DAG.getNode(
749 DL,
750 MVT::i1,
751 Op, DAG.getConstantFP(1.0f, DL, MVT::f32),
753}
754
755SDValue R600TargetLowering::lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const {
756 SDLoc DL(Op);
757 return DAG.getNode(
759 DL,
760 MVT::i1,
761 Op, DAG.getConstantFP(-1.0f, DL, MVT::f32),
763}
764
765SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT,
766 const SDLoc &DL,
767 unsigned DwordOffset) const {
768 unsigned ByteOffset = DwordOffset * 4;
769 PointerType *PtrType =
771
772 // We shouldn't be using an offset wider than 16-bits for implicit parameters.
773 assert(isInt<16>(ByteOffset));
774
775 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
776 DAG.getConstant(ByteOffset, DL, MVT::i32), // PTR
777 MachinePointerInfo(ConstantPointerNull::get(PtrType)));
778}
779
780bool R600TargetLowering::isZero(SDValue Op) const {
781 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op))
782 return Cst->isZero();
783 if (ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Op))
784 return CstFP->isZero();
785 return false;
786}
787
788bool R600TargetLowering::isHWTrueValue(SDValue Op) const {
789 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) {
790 return CFP->isOne();
791 }
792 return isAllOnesConstant(Op);
793}
794
795bool R600TargetLowering::isHWFalseValue(SDValue Op) const {
796 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) {
797 return CFP->getValueAPF().isZero();
798 }
799 return isNullConstant(Op);
800}
801
802SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
803 SDLoc DL(Op);
804 EVT VT = Op.getValueType();
805
806 SDValue LHS = Op.getOperand(0);
807 SDValue RHS = Op.getOperand(1);
808 SDValue True = Op.getOperand(2);
809 SDValue False = Op.getOperand(3);
810 SDValue CC = Op.getOperand(4);
811 SDValue Temp;
812
813 if (VT == MVT::f32) {
814 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
815 SDValue MinMax = combineFMinMaxLegacy(DL, VT, LHS, RHS, True, False, CC,
816 SDNodeFlags(), DCI);
817 if (MinMax)
818 return MinMax;
819 }
820
821 // LHS and RHS are guaranteed to be the same value type
822 EVT CompareVT = LHS.getValueType();
823
824 // Check if we can lower this to a native operation.
825
826 // Try to lower to a SET* instruction:
827 //
828 // SET* can match the following patterns:
829 //
830 // select_cc f32, f32, -1, 0, cc_supported
831 // select_cc f32, f32, 1.0f, 0.0f, cc_supported
832 // select_cc i32, i32, -1, 0, cc_supported
833 //
834
835 // Move hardware True/False values to the correct operand.
836 if (isHWTrueValue(False) && isHWFalseValue(True)) {
837 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
838 ISD::CondCode InverseCC = ISD::getSetCCInverse(CCOpcode, CompareVT);
839 if (isCondCodeLegal(InverseCC, CompareVT.getSimpleVT())) {
841 CC = DAG.getCondCode(InverseCC);
842 } else {
843 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InverseCC);
844 if (isCondCodeLegal(SwapInvCC, CompareVT.getSimpleVT())) {
846 std::swap(LHS, RHS);
847 CC = DAG.getCondCode(SwapInvCC);
848 }
849 }
850 }
851
852 if (isHWTrueValue(True) && isHWFalseValue(False) &&
853 (CompareVT == VT || VT == MVT::i32)) {
854 // This can be matched by a SET* instruction.
855 return DAG.getNode(ISD::SELECT_CC, DL, VT, LHS, RHS, True, False, CC);
856 }
857
858 // Try to lower to a CND* instruction:
859 //
860 // CND* can match the following patterns:
861 //
862 // select_cc f32, 0.0, f32, f32, cc_supported
863 // select_cc f32, 0.0, i32, i32, cc_supported
864 // select_cc i32, 0, f32, f32, cc_supported
865 // select_cc i32, 0, i32, i32, cc_supported
866 //
867
868 // Try to move the zero value to the RHS
869 if (isZero(LHS)) {
870 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
871 // Try swapping the operands
872 ISD::CondCode CCSwapped = ISD::getSetCCSwappedOperands(CCOpcode);
873 if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) {
874 std::swap(LHS, RHS);
875 CC = DAG.getCondCode(CCSwapped);
876 } else {
877 // Try inverting the condition and then swapping the operands
878 ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT);
879 CCSwapped = ISD::getSetCCSwappedOperands(CCInv);
880 if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) {
882 std::swap(LHS, RHS);
883 CC = DAG.getCondCode(CCSwapped);
884 }
885 }
886 }
887 if (isZero(RHS)) {
888 SDValue Cond = LHS;
889 SDValue Zero = RHS;
890 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
891 if (CompareVT != VT) {
892 // Bitcast True / False to the correct types. This will end up being
893 // a nop, but it allows us to define only a single pattern in the
894 // .TD files for each CND* instruction rather than having to have
895 // one pattern for integer True/False and one for fp True/False
896 True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True);
897 False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False);
898 }
899
900 switch (CCOpcode) {
901 case ISD::SETONE:
902 case ISD::SETUNE:
903 case ISD::SETNE:
904 CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT);
905 Temp = True;
906 True = False;
907 False = Temp;
908 break;
909 default:
910 break;
911 }
912 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT,
913 Cond, Zero,
914 True, False,
915 DAG.getCondCode(CCOpcode));
916 return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode);
917 }
918
919 // If we make it this for it means we have no native instructions to handle
920 // this SELECT_CC, so we must lower it.
921 SDValue HWTrue, HWFalse;
922
923 if (CompareVT == MVT::f32) {
924 HWTrue = DAG.getConstantFP(1.0f, DL, CompareVT);
925 HWFalse = DAG.getConstantFP(0.0f, DL, CompareVT);
926 } else if (CompareVT == MVT::i32) {
927 HWTrue = DAG.getAllOnesConstant(DL, CompareVT);
928 HWFalse = DAG.getConstant(0, DL, CompareVT);
929 }
930 else {
931 llvm_unreachable("Unhandled value type in LowerSELECT_CC");
932 }
933
934 // Lower this unsupported SELECT_CC into a combination of two supported
935 // SELECT_CC operations.
936 SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC);
937
938 return DAG.getNode(ISD::SELECT_CC, DL, VT,
939 Cond, HWFalse,
940 True, False,
942}
943
944SDValue R600TargetLowering::lowerADDRSPACECAST(SDValue Op,
945 SelectionDAG &DAG) const {
946 SDLoc SL(Op);
947 EVT VT = Op.getValueType();
948
949 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Op);
950 unsigned SrcAS = ASC->getSrcAddressSpace();
951 unsigned DestAS = ASC->getDestAddressSpace();
952
953 if (isNullConstant(Op.getOperand(0)) && SrcAS == AMDGPUAS::FLAT_ADDRESS)
954 return DAG.getSignedConstant(AMDGPU::getNullPointerValue(DestAS), SL, VT);
955
956 return Op;
957}
958
959SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store,
960 SelectionDAG &DAG) const {
961 SDLoc DL(Store);
962 //TODO: Who creates the i8 stores?
963 assert(Store->isTruncatingStore()
964 || Store->getValue().getValueType() == MVT::i8);
965 assert(Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS);
966
968 if (Store->getMemoryVT() == MVT::i8) {
969 assert(Store->getAlign() >= 1);
970 Mask = DAG.getConstant(0xff, DL, MVT::i32);
971 } else if (Store->getMemoryVT() == MVT::i16) {
972 assert(Store->getAlign() >= 2);
973 Mask = DAG.getConstant(0xffff, DL, MVT::i32);
974 } else {
975 llvm_unreachable("Unsupported private trunc store");
976 }
977
978 SDValue OldChain = Store->getChain();
979 bool VectorTrunc = (OldChain.getOpcode() == AMDGPUISD::DUMMY_CHAIN);
980 // Skip dummy
981 SDValue Chain = VectorTrunc ? OldChain->getOperand(0) : OldChain;
982 SDValue BasePtr = Store->getBasePtr();
983 SDValue Offset = Store->getOffset();
984 EVT MemVT = Store->getMemoryVT();
985
986 SDValue LoadPtr = BasePtr;
987 if (!Offset.isUndef()) {
988 LoadPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, Offset);
989 }
990
991 // Get dword location
992 // TODO: this should be eliminated by the future SHR ptr, 2
993 SDValue Ptr = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
994 DAG.getConstant(0xfffffffc, DL, MVT::i32));
995
996 // Load dword
997 // TODO: can we be smarter about machine pointer info?
998 MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS);
999 SDValue Dst = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo);
1000
1001 Chain = Dst.getValue(1);
1002
1003 // Get offset in dword
1004 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
1005 DAG.getConstant(0x3, DL, MVT::i32));
1006
1007 // Convert byte offset to bit shift
1008 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1009 DAG.getConstant(3, DL, MVT::i32));
1010
1011 // TODO: Contrary to the name of the function,
1012 // it also handles sub i32 non-truncating stores (like i1)
1013 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32,
1014 Store->getValue());
1015
1016 // Mask the value to the right type
1017 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT);
1018
1019 // Shift the value in place
1020 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32,
1021 MaskedValue, ShiftAmt);
1022
1023 // Shift the mask in place
1024 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, Mask, ShiftAmt);
1025
1026 // Invert the mask. NOTE: if we had native ROL instructions we could
1027 // use inverted mask
1028 DstMask = DAG.getNOT(DL, DstMask, MVT::i32);
1029
1030 // Cleanup the target bits
1031 Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask);
1032
1033 // Add the new bits
1034 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue);
1035
1036 // Store dword
1037 // TODO: Can we be smarter about MachinePointerInfo?
1038 SDValue NewStore = DAG.getStore(Chain, DL, Value, Ptr, PtrInfo);
1039
1040 // If we are part of expanded vector, make our neighbors depend on this store
1041 if (VectorTrunc) {
1042 // Make all other vector elements depend on this store
1043 Chain = DAG.getNode(AMDGPUISD::DUMMY_CHAIN, DL, MVT::Other, NewStore);
1044 DAG.ReplaceAllUsesOfValueWith(OldChain, Chain);
1045 }
1046 return NewStore;
1047}
1048
1049SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1050 StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
1051 unsigned AS = StoreNode->getAddressSpace();
1052
1053 SDValue Chain = StoreNode->getChain();
1054 SDValue Ptr = StoreNode->getBasePtr();
1055 SDValue Value = StoreNode->getValue();
1056
1057 EVT VT = Value.getValueType();
1058 EVT MemVT = StoreNode->getMemoryVT();
1059 EVT PtrVT = Ptr.getValueType();
1060
1061 SDLoc DL(Op);
1062
1063 const bool TruncatingStore = StoreNode->isTruncatingStore();
1064
1065 // Neither LOCAL nor PRIVATE can do vectors at the moment
1067 TruncatingStore) &&
1068 VT.isVector()) {
1069 if ((AS == AMDGPUAS::PRIVATE_ADDRESS) && TruncatingStore) {
1070 // Add an extra level of chain to isolate this vector
1071 SDValue NewChain = DAG.getNode(AMDGPUISD::DUMMY_CHAIN, DL, MVT::Other, Chain);
1072 SmallVector<SDValue, 4> NewOps(StoreNode->ops());
1073 NewOps[0] = NewChain;
1074 StoreNode = cast<StoreSDNode>(DAG.UpdateNodeOperands(StoreNode, NewOps));
1075 }
1076
1077 return scalarizeVectorStore(StoreNode, DAG);
1078 }
1079
1080 Align Alignment = StoreNode->getAlign();
1081 if (Alignment < MemVT.getStoreSize() &&
1082 !allowsMisalignedMemoryAccesses(MemVT, AS, Alignment,
1083 StoreNode->getMemOperand()->getFlags(),
1084 nullptr)) {
1085 return expandUnalignedStore(StoreNode, DAG);
1086 }
1087
1088 SDValue DWordAddr = DAG.getNode(ISD::SRL, DL, PtrVT, Ptr,
1089 DAG.getConstant(2, DL, PtrVT));
1090
1091 if (AS == AMDGPUAS::GLOBAL_ADDRESS) {
1092 // It is beneficial to create MSKOR here instead of combiner to avoid
1093 // artificial dependencies introduced by RMW
1094 if (TruncatingStore) {
1095 assert(VT.bitsLE(MVT::i32));
1096 SDValue MaskConstant;
1097 if (MemVT == MVT::i8) {
1098 MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32);
1099 } else {
1100 assert(MemVT == MVT::i16);
1101 assert(StoreNode->getAlign() >= 2);
1102 MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32);
1103 }
1104
1105 SDValue ByteIndex = DAG.getNode(ISD::AND, DL, PtrVT, Ptr,
1106 DAG.getConstant(0x00000003, DL, PtrVT));
1107 SDValue BitShift = DAG.getNode(ISD::SHL, DL, VT, ByteIndex,
1108 DAG.getConstant(3, DL, VT));
1109
1110 // Put the mask in correct place
1111 SDValue Mask = DAG.getNode(ISD::SHL, DL, VT, MaskConstant, BitShift);
1112
1113 // Put the value bits in correct place
1114 SDValue TruncValue = DAG.getNode(ISD::AND, DL, VT, Value, MaskConstant);
1115 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, VT, TruncValue, BitShift);
1116
1117 // XXX: If we add a 64-bit ZW register class, then we could use a 2 x i32
1118 // vector instead.
1119 SDValue Src[4] = {
1120 ShiftedValue,
1121 DAG.getConstant(0, DL, MVT::i32),
1122 DAG.getConstant(0, DL, MVT::i32),
1123 Mask
1124 };
1125 SDValue Input = DAG.getBuildVector(MVT::v4i32, DL, Src);
1126 SDValue Args[3] = { Chain, Input, DWordAddr };
1127 return DAG.getMemIntrinsicNode(AMDGPUISD::STORE_MSKOR, DL,
1128 Op->getVTList(), Args, MemVT,
1129 StoreNode->getMemOperand());
1130 }
1131 if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && VT.bitsGE(MVT::i32)) {
1132 // Convert pointer from byte address to dword address.
1133 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, PtrVT, DWordAddr);
1134
1135 if (StoreNode->isIndexed()) {
1136 llvm_unreachable("Indexed stores not supported yet");
1137 } else {
1138 Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand());
1139 }
1140 return Chain;
1141 }
1142 }
1143
1144 // GLOBAL_ADDRESS has been handled above, LOCAL_ADDRESS allows all sizes
1145 if (AS != AMDGPUAS::PRIVATE_ADDRESS)
1146 return SDValue();
1147
1148 if (MemVT.bitsLT(MVT::i32))
1149 return lowerPrivateTruncStore(StoreNode, DAG);
1150
1151 // Standard i32+ store, tag it with DWORDADDR to note that the address
1152 // has been shifted
1153 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1154 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, PtrVT, DWordAddr);
1155 return DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand());
1156 }
1157
1158 // Tagged i32+ stores will be matched by patterns
1159 return SDValue();
1160}
1161
1162// return (512 + (kc_bank << 12)
1163static int
1165 switch (AddressSpace) {
1167 return 512;
1169 return 512 + 4096;
1171 return 512 + 4096 * 2;
1173 return 512 + 4096 * 3;
1175 return 512 + 4096 * 4;
1177 return 512 + 4096 * 5;
1179 return 512 + 4096 * 6;
1181 return 512 + 4096 * 7;
1183 return 512 + 4096 * 8;
1185 return 512 + 4096 * 9;
1187 return 512 + 4096 * 10;
1189 return 512 + 4096 * 11;
1191 return 512 + 4096 * 12;
1193 return 512 + 4096 * 13;
1195 return 512 + 4096 * 14;
1197 return 512 + 4096 * 15;
1198 default:
1199 return -1;
1200 }
1201}
1202
1203SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op,
1204 SelectionDAG &DAG) const {
1205 SDLoc DL(Op);
1206 LoadSDNode *Load = cast<LoadSDNode>(Op);
1207 ISD::LoadExtType ExtType = Load->getExtensionType();
1208 EVT MemVT = Load->getMemoryVT();
1209 assert(Load->getAlign() >= MemVT.getStoreSize());
1210
1211 SDValue BasePtr = Load->getBasePtr();
1212 SDValue Chain = Load->getChain();
1213 SDValue Offset = Load->getOffset();
1214
1215 SDValue LoadPtr = BasePtr;
1216 if (!Offset.isUndef()) {
1217 LoadPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr, Offset);
1218 }
1219
1220 // Get dword location
1221 // NOTE: this should be eliminated by the future SHR ptr, 2
1222 SDValue Ptr = DAG.getNode(ISD::AND, DL, MVT::i32, LoadPtr,
1223 DAG.getConstant(0xfffffffc, DL, MVT::i32));
1224
1225 // Load dword
1226 // TODO: can we be smarter about machine pointer info?
1227 MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS);
1228 SDValue Read = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo);
1229
1230 // Get offset within the register.
1231 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32,
1232 LoadPtr, DAG.getConstant(0x3, DL, MVT::i32));
1233
1234 // Bit offset of target byte (byteIdx * 8).
1235 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1236 DAG.getConstant(3, DL, MVT::i32));
1237
1238 // Shift to the right.
1239 SDValue Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Read, ShiftAmt);
1240
1241 // Eliminate the upper bits by setting them to ...
1242 EVT MemEltVT = MemVT.getScalarType();
1243
1244 if (ExtType == ISD::SEXTLOAD) { // ... ones.
1245 SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
1246 Ret = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode);
1247 } else { // ... or zeros.
1248 Ret = DAG.getZeroExtendInReg(Ret, DL, MemEltVT);
1249 }
1250
1251 SDValue Ops[] = {
1252 Ret,
1253 Read.getValue(1) // This should be our output chain
1254 };
1255
1256 return DAG.getMergeValues(Ops, DL);
1257}
1258
1259SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1260 LoadSDNode *LoadNode = cast<LoadSDNode>(Op);
1261 unsigned AS = LoadNode->getAddressSpace();
1262 EVT MemVT = LoadNode->getMemoryVT();
1263 ISD::LoadExtType ExtType = LoadNode->getExtensionType();
1264
1265 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
1266 ExtType != ISD::NON_EXTLOAD && MemVT.bitsLT(MVT::i32)) {
1267 return lowerPrivateExtLoad(Op, DAG);
1268 }
1269
1270 SDLoc DL(Op);
1271 EVT VT = Op.getValueType();
1272 SDValue Chain = LoadNode->getChain();
1273 SDValue Ptr = LoadNode->getBasePtr();
1274
1275 if ((LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1277 VT.isVector()) {
1278 SDValue Ops[2];
1279 std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(LoadNode, DAG);
1280 return DAG.getMergeValues(Ops, DL);
1281 }
1282
1283 // This is still used for explicit load from addrspace(8)
1284 int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace());
1285 if (ConstantBlock > -1 &&
1286 ((LoadNode->getExtensionType() == ISD::NON_EXTLOAD) ||
1287 (LoadNode->getExtensionType() == ISD::ZEXTLOAD))) {
1289 if (isa<Constant>(LoadNode->getMemOperand()->getValue()) ||
1290 isa<ConstantSDNode>(Ptr)) {
1291 return constBufferLoad(LoadNode, LoadNode->getAddressSpace(), DAG);
1292 }
1293 // TODO: Does this even work?
1294 // non-constant ptr can't be folded, keeps it as a v4f32 load
1295 Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32,
1296 DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
1297 DAG.getConstant(4, DL, MVT::i32)),
1298 DAG.getConstant(LoadNode->getAddressSpace() -
1300 DL, MVT::i32));
1301
1302 if (!VT.isVector()) {
1303 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result,
1304 DAG.getConstant(0, DL, MVT::i32));
1305 }
1306
1307 SDValue MergedValues[2] = {
1308 Result,
1309 Chain
1310 };
1311 return DAG.getMergeValues(MergedValues, DL);
1312 }
1313
1314 // For most operations returning SDValue() will result in the node being
1315 // expanded by the DAG Legalizer. This is not the case for ISD::LOAD, so we
1316 // need to manually expand loads that may be legal in some address spaces and
1317 // illegal in others. SEXT loads from CONSTANT_BUFFER_0 are supported for
1318 // compute shaders, since the data is sign extended when it is uploaded to the
1319 // buffer. However SEXT loads from other address spaces are not supported, so
1320 // we need to expand them here.
1321 if (LoadNode->getExtensionType() == ISD::SEXTLOAD) {
1322 assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8));
1323 SDValue NewLoad = DAG.getExtLoad(
1324 ISD::EXTLOAD, DL, VT, Chain, Ptr, LoadNode->getPointerInfo(), MemVT,
1325 LoadNode->getAlign(), LoadNode->getMemOperand()->getFlags());
1326 SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewLoad,
1327 DAG.getValueType(MemVT));
1328
1329 SDValue MergedValues[2] = { Res, Chain };
1330 return DAG.getMergeValues(MergedValues, DL);
1331 }
1332
1333 if (LoadNode->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) {
1334 return SDValue();
1335 }
1336
1337 // DWORDADDR ISD marks already shifted address
1338 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1339 assert(VT == MVT::i32);
1340 Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, DAG.getConstant(2, DL, MVT::i32));
1341 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, MVT::i32, Ptr);
1342 return DAG.getLoad(MVT::i32, DL, Chain, Ptr, LoadNode->getMemOperand());
1343 }
1344 return SDValue();
1345}
1346
1347SDValue R600TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
1348 SDValue Chain = Op.getOperand(0);
1349 SDValue Cond = Op.getOperand(1);
1350 SDValue Jump = Op.getOperand(2);
1351
1352 return DAG.getNode(AMDGPUISD::BRANCH_COND, SDLoc(Op), Op.getValueType(),
1353 Chain, Jump, Cond);
1354}
1355
1356SDValue R600TargetLowering::lowerFrameIndex(SDValue Op,
1357 SelectionDAG &DAG) const {
1359 const R600FrameLowering *TFL = Subtarget->getFrameLowering();
1360
1361 FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op);
1362
1363 unsigned FrameIndex = FIN->getIndex();
1364 Register IgnoredFrameReg;
1365 StackOffset Offset =
1366 TFL->getFrameIndexReference(MF, FrameIndex, IgnoredFrameReg);
1367 return DAG.getConstant(Offset.getFixed() * 4 * TFL->getStackWidth(MF),
1368 SDLoc(Op), Op.getValueType());
1369}
1370
1372 bool IsVarArg) const {
1373 switch (CC) {
1376 case CallingConv::C:
1377 case CallingConv::Fast:
1378 case CallingConv::Cold:
1379 llvm_unreachable("kernels should not be handled here");
1387 return CC_R600;
1388 default:
1389 reportFatalUsageError("unsupported calling convention");
1390 }
1391}
1392
1393/// XXX Only kernel functions are supported, so we can assume for now that
1394/// every function is a kernel function, but in the future we should use
1395/// separate calling conventions for kernel and non-kernel functions.
1397 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1398 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1399 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1401 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
1402 *DAG.getContext());
1404
1405 if (AMDGPU::isShader(CallConv)) {
1406 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg));
1407 } else {
1408 analyzeFormalArgumentsCompute(CCInfo, Ins);
1409 }
1410
1411 for (unsigned i = 0, e = Ins.size(); i < e; ++i) {
1412 CCValAssign &VA = ArgLocs[i];
1413 const ISD::InputArg &In = Ins[i];
1414 EVT VT = In.VT;
1415 EVT MemVT = VA.getLocVT();
1416 if (!VT.isVector() && MemVT.isVector()) {
1417 // Get load source type if scalarized.
1418 MemVT = MemVT.getVectorElementType();
1419 }
1420
1421 if (VT.isInteger() && !MemVT.isInteger())
1422 MemVT = MemVT.changeTypeToInteger();
1423
1424 if (AMDGPU::isShader(CallConv)) {
1425 Register Reg = MF.addLiveIn(VA.getLocReg(), &R600::R600_Reg128RegClass);
1426 SDValue Register = DAG.getCopyFromReg(Chain, DL, Reg, VT);
1427 InVals.push_back(Register);
1428 continue;
1429 }
1430
1431 // i64 isn't a legal type, so the register type used ends up as i32, which
1432 // isn't expected here. It attempts to create this sextload, but it ends up
1433 // being invalid. Somehow this seems to work with i64 arguments, but breaks
1434 // for <1 x i64>.
1435
1436 // The first 36 bytes of the input buffer contains information about
1437 // thread group and global sizes.
1439 if (MemVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) {
1440 if (VT.isFloatingPoint()) {
1441 Ext = ISD::EXTLOAD;
1442 } else {
1443 // FIXME: This should really check the extload type, but the handling of
1444 // extload vector parameters seems to be broken.
1445
1446 // Ext = In.Flags.isSExt() ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
1447 Ext = ISD::SEXTLOAD;
1448 }
1449 }
1450
1451 // Compute the offset from the value.
1452 // XXX - I think PartOffset should give you this, but it seems to give the
1453 // size of the register which isn't useful.
1454
1455 unsigned PartOffset = VA.getLocMemOffset();
1456 Align Alignment = commonAlignment(Align(VT.getStoreSize()), PartOffset);
1457
1459 SDValue Arg =
1460 DAG.getLoad(ISD::UNINDEXED, Ext, VT, DL, Chain,
1461 DAG.getConstant(PartOffset, DL, MVT::i32),
1462 DAG.getPOISON(MVT::i32), PtrInfo, MemVT, Alignment,
1466
1467 InVals.push_back(Arg);
1468 }
1469 return Chain;
1470}
1471
1473 EVT VT) const {
1474 if (!VT.isVector())
1475 return MVT::i32;
1477}
1478
1480 const MachineFunction &MF) const {
1481 // Local and Private addresses do not handle vectors. Limit to i32
1483 return (MemVT.getSizeInBits() <= 32);
1484 }
1485 return true;
1486}
1487
1489 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1490 unsigned *IsFast) const {
1491 if (IsFast)
1492 *IsFast = 0;
1493
1494 if (!VT.isSimple() || VT == MVT::Other)
1495 return false;
1496
1497 if (VT.bitsLT(MVT::i32))
1498 return false;
1499
1500 // TODO: This is a rough estimate.
1501 if (IsFast)
1502 *IsFast = 1;
1503
1504 return VT.bitsGT(MVT::i32) && Alignment >= Align(4);
1505}
1506
1508 SelectionDAG &DAG, SDValue VectorEntry,
1509 DenseMap<unsigned, unsigned> &RemapSwizzle) {
1510 assert(RemapSwizzle.empty());
1511
1512 SDLoc DL(VectorEntry);
1513 EVT EltTy = VectorEntry.getValueType().getVectorElementType();
1514
1515 SDValue NewBldVec[4];
1516 for (unsigned i = 0; i < 4; i++)
1517 NewBldVec[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltTy, VectorEntry,
1518 DAG.getIntPtrConstant(i, DL));
1519
1520 for (unsigned i = 0; i < 4; i++) {
1521 if (NewBldVec[i].isUndef())
1522 // We mask write here to teach later passes that the ith element of this
1523 // vector is undef. Thus we can use it to reduce 128 bits reg usage,
1524 // break false dependencies and additionally make assembly easier to read.
1525 RemapSwizzle[i] = 7; // SEL_MASK_WRITE
1526 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(NewBldVec[i])) {
1527 if (C->isZero()) {
1528 RemapSwizzle[i] = 4; // SEL_0
1529 NewBldVec[i] = DAG.getUNDEF(MVT::f32);
1530 } else if (C->isOne()) {
1531 RemapSwizzle[i] = 5; // SEL_1
1532 NewBldVec[i] = DAG.getUNDEF(MVT::f32);
1533 }
1534 }
1535
1536 if (NewBldVec[i].isUndef())
1537 continue;
1538
1539 for (unsigned j = 0; j < i; j++) {
1540 if (NewBldVec[i] == NewBldVec[j]) {
1541 NewBldVec[i] = DAG.getUNDEF(NewBldVec[i].getValueType());
1542 RemapSwizzle[i] = j;
1543 break;
1544 }
1545 }
1546 }
1547
1548 return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry),
1549 NewBldVec);
1550}
1551
1553 DenseMap<unsigned, unsigned> &RemapSwizzle) {
1554 assert(RemapSwizzle.empty());
1555
1556 SDLoc DL(VectorEntry);
1557 EVT EltTy = VectorEntry.getValueType().getVectorElementType();
1558
1559 SDValue NewBldVec[4];
1560 bool isUnmovable[4] = {false, false, false, false};
1561 for (unsigned i = 0; i < 4; i++)
1562 NewBldVec[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltTy, VectorEntry,
1563 DAG.getIntPtrConstant(i, DL));
1564
1565 for (unsigned i = 0; i < 4; i++) {
1566 RemapSwizzle[i] = i;
1567 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1568 unsigned Idx = NewBldVec[i].getConstantOperandVal(1);
1569 if (i == Idx)
1570 isUnmovable[Idx] = true;
1571 }
1572 }
1573
1574 for (unsigned i = 0; i < 4; i++) {
1575 if (NewBldVec[i].getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1576 unsigned Idx = NewBldVec[i].getConstantOperandVal(1);
1577 if (isUnmovable[Idx])
1578 continue;
1579 // Swap i and Idx
1580 std::swap(NewBldVec[Idx], NewBldVec[i]);
1581 std::swap(RemapSwizzle[i], RemapSwizzle[Idx]);
1582 break;
1583 }
1584 }
1585
1586 return DAG.getBuildVector(VectorEntry.getValueType(), SDLoc(VectorEntry),
1587 NewBldVec);
1588}
1589
1590SDValue R600TargetLowering::OptimizeSwizzle(SDValue BuildVector, SDValue Swz[],
1591 SelectionDAG &DAG,
1592 const SDLoc &DL) const {
1593 // Old -> New swizzle values
1594 DenseMap<unsigned, unsigned> SwizzleRemap;
1595
1596 BuildVector = CompactSwizzlableVector(DAG, BuildVector, SwizzleRemap);
1597 for (unsigned i = 0; i < 4; i++) {
1598 unsigned Idx = Swz[i]->getAsZExtVal();
1599 auto It = SwizzleRemap.find(Idx);
1600 if (It != SwizzleRemap.end())
1601 Swz[i] = DAG.getConstant(It->second, DL, MVT::i32);
1602 }
1603
1604 SwizzleRemap.clear();
1605 BuildVector = ReorganizeVector(DAG, BuildVector, SwizzleRemap);
1606 for (unsigned i = 0; i < 4; i++) {
1607 unsigned Idx = Swz[i]->getAsZExtVal();
1608 auto It = SwizzleRemap.find(Idx);
1609 if (It != SwizzleRemap.end())
1610 Swz[i] = DAG.getConstant(It->second, DL, MVT::i32);
1611 }
1612
1613 return BuildVector;
1614}
1615
1616SDValue R600TargetLowering::constBufferLoad(LoadSDNode *LoadNode, int Block,
1617 SelectionDAG &DAG) const {
1618 SDLoc DL(LoadNode);
1619 EVT VT = LoadNode->getValueType(0);
1620 SDValue Chain = LoadNode->getChain();
1621 SDValue Ptr = LoadNode->getBasePtr();
1623
1624 //TODO: Support smaller loads
1625 if (LoadNode->getMemoryVT().getScalarType() != MVT::i32 || !ISD::isNON_EXTLoad(LoadNode))
1626 return SDValue();
1627
1628 if (LoadNode->getAlign() < Align(4))
1629 return SDValue();
1630
1631 int ConstantBlock = ConstantAddressBlock(Block);
1632
1633 SDValue Slots[4];
1634 for (unsigned i = 0; i < 4; i++) {
1635 // We want Const position encoded with the following formula :
1636 // (((512 + (kc_bank << 12) + const_index) << 2) + chan)
1637 // const_index is Ptr computed by llvm using an alignment of 16.
1638 // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and
1639 // then div by 4 at the ISel step
1640 SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
1641 DAG.getConstant(4 * i + ConstantBlock * 16, DL, MVT::i32));
1642 Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr);
1643 }
1644 EVT NewVT = MVT::v4i32;
1645 unsigned NumElements = 4;
1646 if (VT.isVector()) {
1647 NewVT = VT;
1648 NumElements = VT.getVectorNumElements();
1649 }
1650 SDValue Result = DAG.getBuildVector(NewVT, DL, ArrayRef(Slots, NumElements));
1651 if (!VT.isVector()) {
1652 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result,
1653 DAG.getConstant(0, DL, MVT::i32));
1654 }
1655 SDValue MergedValues[2] = {
1656 Result,
1657 Chain
1658 };
1659 return DAG.getMergeValues(MergedValues, DL);
1660}
1661
1662//===----------------------------------------------------------------------===//
1663// Custom DAG Optimizations
1664//===----------------------------------------------------------------------===//
1665
1667 DAGCombinerInfo &DCI) const {
1668 SelectionDAG &DAG = DCI.DAG;
1669 SDLoc DL(N);
1670
1671 switch (N->getOpcode()) {
1672 // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a)
1673 case ISD::FP_ROUND: {
1674 SDValue Arg = N->getOperand(0);
1675 if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) {
1676 return DAG.getNode(ISD::UINT_TO_FP, DL, N->getValueType(0),
1677 Arg.getOperand(0));
1678 }
1679 break;
1680 }
1681
1682 // (i32 fp_to_sint (fneg (select_cc f32, f32, 1.0, 0.0 cc))) ->
1683 // (i32 select_cc f32, f32, -1, 0 cc)
1684 //
1685 // Mesa's GLSL frontend generates the above pattern a lot and we can lower
1686 // this to one of the SET*_DX10 instructions.
1687 case ISD::FP_TO_SINT: {
1688 SDValue FNeg = N->getOperand(0);
1689 if (FNeg.getOpcode() != ISD::FNEG) {
1690 return SDValue();
1691 }
1692 SDValue SelectCC = FNeg.getOperand(0);
1693 if (SelectCC.getOpcode() != ISD::SELECT_CC ||
1694 SelectCC.getOperand(0).getValueType() != MVT::f32 || // LHS
1695 SelectCC.getOperand(2).getValueType() != MVT::f32 || // True
1696 !isHWTrueValue(SelectCC.getOperand(2)) ||
1697 !isHWFalseValue(SelectCC.getOperand(3))) {
1698 return SDValue();
1699 }
1700
1701 return DAG.getNode(ISD::SELECT_CC, DL, N->getValueType(0),
1702 SelectCC.getOperand(0), // LHS
1703 SelectCC.getOperand(1), // RHS
1704 DAG.getAllOnesConstant(DL, MVT::i32), // True
1705 DAG.getConstant(0, DL, MVT::i32), // False
1706 SelectCC.getOperand(4)); // CC
1707 }
1708
1709 // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx
1710 // => build_vector elt0, ... , NewEltIdx, ... , eltN
1712 SDValue InVec = N->getOperand(0);
1713 SDValue InVal = N->getOperand(1);
1714 SDValue EltNo = N->getOperand(2);
1715
1716 // If the inserted element is an UNDEF, just use the input vector.
1717 if (InVal.isUndef())
1718 return InVec;
1719
1720 EVT VT = InVec.getValueType();
1721
1722 // If we can't generate a legal BUILD_VECTOR, exit
1724 return SDValue();
1725
1726 // Check that we know which element is being inserted
1727 if (!isa<ConstantSDNode>(EltNo))
1728 return SDValue();
1729 unsigned Elt = EltNo->getAsZExtVal();
1730
1731 // Check that the operand is a BUILD_VECTOR (or UNDEF, which can essentially
1732 // be converted to a BUILD_VECTOR). Fill in the Ops vector with the
1733 // vector elements.
1735 if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
1736 Ops.append(InVec.getNode()->op_begin(),
1737 InVec.getNode()->op_end());
1738 } else if (InVec.isUndef()) {
1739 unsigned NElts = VT.getVectorNumElements();
1740 Ops.append(NElts, DAG.getUNDEF(InVal.getValueType()));
1741 } else {
1742 return SDValue();
1743 }
1744
1745 // Insert the element
1746 if (Elt < Ops.size()) {
1747 // All the operands of BUILD_VECTOR must have the same type;
1748 // we enforce that here.
1749 EVT OpVT = Ops[0].getValueType();
1750 if (InVal.getValueType() != OpVT)
1751 InVal = OpVT.bitsGT(InVal.getValueType()) ?
1752 DAG.getNode(ISD::ANY_EXTEND, DL, OpVT, InVal) :
1753 DAG.getNode(ISD::TRUNCATE, DL, OpVT, InVal);
1754 Ops[Elt] = InVal;
1755 }
1756
1757 // Return the new vector
1758 return DAG.getBuildVector(VT, DL, Ops);
1759 }
1760
1761 // Extract_vec (Build_vector) generated by custom lowering
1762 // also needs to be customly combined
1764 SDValue Arg = N->getOperand(0);
1765 if (Arg.getOpcode() == ISD::BUILD_VECTOR) {
1766 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1767 unsigned Element = Const->getZExtValue();
1768 return Arg->getOperand(Element);
1769 }
1770 }
1771 if (Arg.getOpcode() == ISD::BITCAST &&
1775 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1776 unsigned Element = Const->getZExtValue();
1777 return DAG.getNode(ISD::BITCAST, DL, N->getVTList(),
1778 Arg->getOperand(0).getOperand(Element));
1779 }
1780 }
1781 break;
1782 }
1783
1784 case ISD::SELECT_CC: {
1785 // Try common optimizations
1787 return Ret;
1788
1789 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, seteq ->
1790 // selectcc x, y, a, b, inv(cc)
1791 //
1792 // fold selectcc (selectcc x, y, a, b, cc), b, a, b, setne ->
1793 // selectcc x, y, a, b, cc
1794 SDValue LHS = N->getOperand(0);
1795 if (LHS.getOpcode() != ISD::SELECT_CC) {
1796 return SDValue();
1797 }
1798
1799 SDValue RHS = N->getOperand(1);
1800 SDValue True = N->getOperand(2);
1801 SDValue False = N->getOperand(3);
1802 ISD::CondCode NCC = N->getOperandCondCode(4);
1803
1804 if (LHS.getOperand(2).getNode() != True.getNode() ||
1805 LHS.getOperand(3).getNode() != False.getNode() ||
1806 RHS.getNode() != False.getNode()) {
1807 return SDValue();
1808 }
1809
1810 switch (NCC) {
1811 default: return SDValue();
1812 case ISD::SETNE: return LHS;
1813 case ISD::SETEQ: {
1814 ISD::CondCode LHSCC = LHS.getOperandCondCode(4);
1815 LHSCC = ISD::getSetCCInverse(LHSCC, LHS.getOperand(0).getValueType());
1816 if (DCI.isBeforeLegalizeOps() ||
1817 isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType()))
1818 return DAG.getSelectCC(DL,
1819 LHS.getOperand(0),
1820 LHS.getOperand(1),
1821 LHS.getOperand(2),
1822 LHS.getOperand(3),
1823 LHSCC);
1824 break;
1825 }
1826 }
1827 return SDValue();
1828 }
1829
1831 SDValue Arg = N->getOperand(1);
1832 if (Arg.getOpcode() != ISD::BUILD_VECTOR)
1833 break;
1834
1835 SDValue NewArgs[8] = {
1836 N->getOperand(0), // Chain
1837 SDValue(),
1838 N->getOperand(2), // ArrayBase
1839 N->getOperand(3), // Type
1840 N->getOperand(4), // SWZ_X
1841 N->getOperand(5), // SWZ_Y
1842 N->getOperand(6), // SWZ_Z
1843 N->getOperand(7) // SWZ_W
1844 };
1845 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL);
1846 return DAG.getNode(AMDGPUISD::R600_EXPORT, DL, N->getVTList(), NewArgs);
1847 }
1849 SDValue Arg = N->getOperand(1);
1850 if (Arg.getOpcode() != ISD::BUILD_VECTOR)
1851 break;
1852
1853 SDValue NewArgs[19] = {
1854 N->getOperand(0),
1855 N->getOperand(1),
1856 N->getOperand(2),
1857 N->getOperand(3),
1858 N->getOperand(4),
1859 N->getOperand(5),
1860 N->getOperand(6),
1861 N->getOperand(7),
1862 N->getOperand(8),
1863 N->getOperand(9),
1864 N->getOperand(10),
1865 N->getOperand(11),
1866 N->getOperand(12),
1867 N->getOperand(13),
1868 N->getOperand(14),
1869 N->getOperand(15),
1870 N->getOperand(16),
1871 N->getOperand(17),
1872 N->getOperand(18),
1873 };
1874 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL);
1875 return DAG.getNode(AMDGPUISD::TEXTURE_FETCH, DL, N->getVTList(), NewArgs);
1876 }
1877
1878 case ISD::LOAD: {
1879 LoadSDNode *LoadNode = cast<LoadSDNode>(N);
1880 SDValue Ptr = LoadNode->getBasePtr();
1881 if (LoadNode->getAddressSpace() == AMDGPUAS::PARAM_I_ADDRESS &&
1883 return constBufferLoad(LoadNode, AMDGPUAS::CONSTANT_BUFFER_0, DAG);
1884 break;
1885 }
1886
1887 default: break;
1888 }
1889
1891}
1892
1893bool R600TargetLowering::FoldOperand(SDNode *ParentNode, unsigned SrcIdx,
1894 SDValue &Src, SDValue &Neg, SDValue &Abs,
1895 SDValue &Sel, SDValue &Imm,
1896 SelectionDAG &DAG) const {
1897 const R600InstrInfo *TII = Subtarget->getInstrInfo();
1898 if (!Src.isMachineOpcode())
1899 return false;
1900
1901 switch (Src.getMachineOpcode()) {
1902 case R600::FNEG_R600:
1903 if (!Neg.getNode())
1904 return false;
1905 Src = Src.getOperand(0);
1906 Neg = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32);
1907 return true;
1908 case R600::FABS_R600:
1909 if (!Abs.getNode())
1910 return false;
1911 Src = Src.getOperand(0);
1912 Abs = DAG.getTargetConstant(1, SDLoc(ParentNode), MVT::i32);
1913 return true;
1914 case R600::CONST_COPY: {
1915 unsigned Opcode = ParentNode->getMachineOpcode();
1916 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
1917
1918 if (!Sel.getNode())
1919 return false;
1920
1921 SDValue CstOffset = Src.getOperand(0);
1922 if (ParentNode->getValueType(0).isVector())
1923 return false;
1924
1925 // Gather constants values
1926 int SrcIndices[] = {
1927 TII->getOperandIdx(Opcode, R600::OpName::src0),
1928 TII->getOperandIdx(Opcode, R600::OpName::src1),
1929 TII->getOperandIdx(Opcode, R600::OpName::src2),
1930 TII->getOperandIdx(Opcode, R600::OpName::src0_X),
1931 TII->getOperandIdx(Opcode, R600::OpName::src0_Y),
1932 TII->getOperandIdx(Opcode, R600::OpName::src0_Z),
1933 TII->getOperandIdx(Opcode, R600::OpName::src0_W),
1934 TII->getOperandIdx(Opcode, R600::OpName::src1_X),
1935 TII->getOperandIdx(Opcode, R600::OpName::src1_Y),
1936 TII->getOperandIdx(Opcode, R600::OpName::src1_Z),
1937 TII->getOperandIdx(Opcode, R600::OpName::src1_W)
1938 };
1939 std::vector<unsigned> Consts;
1940 for (int OtherSrcIdx : SrcIndices) {
1941 int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx);
1942 if (OtherSrcIdx < 0 || OtherSelIdx < 0)
1943 continue;
1944 if (HasDst) {
1945 OtherSrcIdx--;
1946 OtherSelIdx--;
1947 }
1948 if (RegisterSDNode *Reg =
1949 dyn_cast<RegisterSDNode>(ParentNode->getOperand(OtherSrcIdx))) {
1950 if (Reg->getReg() == R600::ALU_CONST) {
1951 Consts.push_back(ParentNode->getConstantOperandVal(OtherSelIdx));
1952 }
1953 }
1954 }
1955
1956 ConstantSDNode *Cst = cast<ConstantSDNode>(CstOffset);
1957 Consts.push_back(Cst->getZExtValue());
1958 if (!TII->fitsConstReadLimitations(Consts)) {
1959 return false;
1960 }
1961
1962 Sel = CstOffset;
1963 Src = DAG.getRegister(R600::ALU_CONST, MVT::f32);
1964 return true;
1965 }
1966 case R600::MOV_IMM_GLOBAL_ADDR:
1967 // Check if the Imm slot is used. Taken from below.
1968 if (Imm->getAsZExtVal())
1969 return false;
1970 Imm = Src.getOperand(0);
1971 Src = DAG.getRegister(R600::ALU_LITERAL_X, MVT::i32);
1972 return true;
1973 case R600::MOV_IMM_I32:
1974 case R600::MOV_IMM_F32: {
1975 unsigned ImmReg = R600::ALU_LITERAL_X;
1976 uint64_t ImmValue = 0;
1977
1978 if (Src.getMachineOpcode() == R600::MOV_IMM_F32) {
1979 ConstantFPSDNode *FPC = cast<ConstantFPSDNode>(Src.getOperand(0));
1980 float FloatValue = FPC->getValueAPF().convertToFloat();
1981 if (FloatValue == 0.0) {
1982 ImmReg = R600::ZERO;
1983 } else if (FloatValue == 0.5) {
1984 ImmReg = R600::HALF;
1985 } else if (FloatValue == 1.0) {
1986 ImmReg = R600::ONE;
1987 } else {
1988 ImmValue = FPC->getValueAPF().bitcastToAPInt().getZExtValue();
1989 }
1990 } else {
1991 uint64_t Value = Src.getConstantOperandVal(0);
1992 if (Value == 0) {
1993 ImmReg = R600::ZERO;
1994 } else if (Value == 1) {
1995 ImmReg = R600::ONE_INT;
1996 } else {
1997 ImmValue = Value;
1998 }
1999 }
2000
2001 // Check that we aren't already using an immediate.
2002 // XXX: It's possible for an instruction to have more than one
2003 // immediate operand, but this is not supported yet.
2004 if (ImmReg == R600::ALU_LITERAL_X) {
2005 if (!Imm.getNode())
2006 return false;
2007 ConstantSDNode *C = cast<ConstantSDNode>(Imm);
2008 if (C->getZExtValue())
2009 return false;
2010 Imm = DAG.getTargetConstant(ImmValue, SDLoc(ParentNode), MVT::i32);
2011 }
2012 Src = DAG.getRegister(ImmReg, MVT::i32);
2013 return true;
2014 }
2015 default:
2016 return false;
2017 }
2018}
2019
2020/// Fold the instructions after selecting them
2021SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
2022 SelectionDAG &DAG) const {
2023 const R600InstrInfo *TII = Subtarget->getInstrInfo();
2024 if (!Node->isMachineOpcode())
2025 return Node;
2026
2027 unsigned Opcode = Node->getMachineOpcode();
2028 SDValue FakeOp;
2029
2030 std::vector<SDValue> Ops(Node->op_begin(), Node->op_end());
2031
2032 if (Opcode == R600::DOT_4) {
2033 int OperandIdx[] = {
2034 TII->getOperandIdx(Opcode, R600::OpName::src0_X),
2035 TII->getOperandIdx(Opcode, R600::OpName::src0_Y),
2036 TII->getOperandIdx(Opcode, R600::OpName::src0_Z),
2037 TII->getOperandIdx(Opcode, R600::OpName::src0_W),
2038 TII->getOperandIdx(Opcode, R600::OpName::src1_X),
2039 TII->getOperandIdx(Opcode, R600::OpName::src1_Y),
2040 TII->getOperandIdx(Opcode, R600::OpName::src1_Z),
2041 TII->getOperandIdx(Opcode, R600::OpName::src1_W)
2042 };
2043 int NegIdx[] = {
2044 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_X),
2045 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Y),
2046 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Z),
2047 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_W),
2048 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_X),
2049 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Y),
2050 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Z),
2051 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_W)
2052 };
2053 int AbsIdx[] = {
2054 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_X),
2055 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Y),
2056 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Z),
2057 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_W),
2058 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_X),
2059 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Y),
2060 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Z),
2061 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_W)
2062 };
2063 for (unsigned i = 0; i < 8; i++) {
2064 if (OperandIdx[i] < 0)
2065 return Node;
2066 SDValue &Src = Ops[OperandIdx[i] - 1];
2067 SDValue &Neg = Ops[NegIdx[i] - 1];
2068 SDValue &Abs = Ops[AbsIdx[i] - 1];
2069 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2070 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]);
2071 if (HasDst)
2072 SelIdx--;
2073 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2074 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, FakeOp, DAG))
2075 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops);
2076 }
2077 } else if (Opcode == R600::REG_SEQUENCE) {
2078 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) {
2079 SDValue &Src = Ops[i];
2080 if (FoldOperand(Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG))
2081 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops);
2082 }
2083 } else {
2084 if (!TII->hasInstrModifiers(Opcode))
2085 return Node;
2086 int OperandIdx[] = {
2087 TII->getOperandIdx(Opcode, R600::OpName::src0),
2088 TII->getOperandIdx(Opcode, R600::OpName::src1),
2089 TII->getOperandIdx(Opcode, R600::OpName::src2)
2090 };
2091 int NegIdx[] = {
2092 TII->getOperandIdx(Opcode, R600::OpName::src0_neg),
2093 TII->getOperandIdx(Opcode, R600::OpName::src1_neg),
2094 TII->getOperandIdx(Opcode, R600::OpName::src2_neg)
2095 };
2096 int AbsIdx[] = {
2097 TII->getOperandIdx(Opcode, R600::OpName::src0_abs),
2098 TII->getOperandIdx(Opcode, R600::OpName::src1_abs),
2099 -1
2100 };
2101 for (unsigned i = 0; i < 3; i++) {
2102 if (OperandIdx[i] < 0)
2103 return Node;
2104 SDValue &Src = Ops[OperandIdx[i] - 1];
2105 SDValue &Neg = Ops[NegIdx[i] - 1];
2106 SDValue FakeAbs;
2107 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs;
2108 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2109 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]);
2110 int ImmIdx = TII->getOperandIdx(Opcode, R600::OpName::literal);
2111 if (HasDst) {
2112 SelIdx--;
2113 ImmIdx--;
2114 }
2115 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2116 SDValue &Imm = Ops[ImmIdx];
2117 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, Imm, DAG))
2118 return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops);
2119 }
2120 }
2121
2122 return Node;
2123}
2124
2126R600TargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
2127 switch (RMW->getOperation()) {
2138 // FIXME: Cayman at least appears to have instructions for this, but the
2139 // instruction definitions appear to be missing.
2141 case AtomicRMWInst::Xchg: {
2142 const DataLayout &DL = RMW->getFunction()->getDataLayout();
2143 unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
2144 if (ValSize == 32 || ValSize == 64)
2147 }
2148 default:
2149 if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) {
2150 unsigned Size = IntTy->getBitWidth();
2151 if (Size == 32 || Size == 64)
2153 }
2154
2156 }
2157
2158 llvm_unreachable("covered atomicrmw op switch");
2159}
return SDValue()
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define I(x, y, z)
Definition MD5.cpp:57
static bool isUndef(const MachineInstr &MI)
Register Reg
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define MO_FLAG_NEG
Definition R600Defines.h:15
#define MO_FLAG_ABS
Definition R600Defines.h:16
#define MO_FLAG_MASK
Definition R600Defines.h:17
#define MO_FLAG_PUSH
Definition R600Defines.h:18
static bool isEOP(MachineBasicBlock::iterator I)
static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry, DenseMap< unsigned, unsigned > &RemapSwizzle)
static int ConstantAddressBlock(unsigned AddressSpace)
static SDValue CompactSwizzlableVector(SelectionDAG &DAG, SDValue VectorEntry, DenseMap< unsigned, unsigned > &RemapSwizzle)
R600 DAG Lowering interface definition.
Provides R600 specific target descriptions.
AMDGPU R600 specific subclass of TargetSubtarget.
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
Value * RHS
Value * LHS
unsigned getStackWidth(const MachineFunction &MF) const
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI, SDValue Op, SelectionDAG &DAG) const
SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const
Flags must be the select flags, not the compare (SELECT_CC flags come from the fcmp and say nothing a...
SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
SDValue CreateLiveInRegisterRaw(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT) const
AMDGPUTargetLowering(const TargetMachine &TM, const TargetSubtargetInfo &STI, const AMDGPUSubtarget &AMDGPUSTI)
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
LLVM_ABI float convertToFloat() const
Converts this APFloat to host float value.
Definition APFloat.cpp:6306
APInt bitcastToAPInt() const
Definition APFloat.h:1475
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned getSrcAddressSpace() const
unsigned getDestAddressSpace() const
an instruction that atomically reads a memory location, combines it with another value,...
@ FAdd
*p = old + v
@ USubCond
Subtract only if no unsigned overflow.
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ FSub
*p = old - v
@ UIncWrap
Increment one up to a maximum value.
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
@ Nand
*p = ~(old & v)
BinOp getOperation() const
CCState - This class holds information needed while lowering arguments and return values.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
int64_t getLocMemOffset() const
const APFloat & getValueAPF() const
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
uint64_t getZExtValue() const
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool empty() const
Definition DenseMap.h:716
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
Definition Function.cpp:356
LLVM_ABI unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Machine Value Type.
static auto integer_valuetypes()
LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
Representation of each machine instruction.
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
Flags getFlags() const
Return the raw flags of the source value,.
const Value * getValue() const
Return the base address of the memory access.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
unsigned getTargetFlags() const
int64_t getOffset() const
Return the offset from the symbol in this operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
Align getAlign() const
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:882
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
R600TargetLowering(const TargetMachine &TM, const R600Subtarget &STI)
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &, EVT VT) const override
Return the ValueType of the result of SETCC operations.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const override
Determine if the target supports unaligned memory accesses.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
XXX Only kernel functions are supported, so we can assume for now that every function is a kernel fun...
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
Primary interface to the complete machine description for the target machine.
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ PARAM_I_ADDRESS
Address space for indirect addressable parameter memory (VTX1).
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUILD_VERTICAL_VECTOR
This node is for VLIW targets and it is used to represent a vector that is stored in consecutive regi...
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
LLVM_READNONE constexpr bool isShader(CallingConv::ID CC)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
Definition CallingConv.h:47
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition CallingConv.h:41
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition ISDOpcodes.h:527
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ GlobalAddress
Definition ISDOpcodes.h:90
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition ISDOpcodes.h:282
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:865
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
int32_t getLDSNoRetOp(uint32_t Opcode)
constexpr float pif
Definition MathExtras.h:54
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Kill
The last use of a register.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
@ AfterLegalizeVectorOps
Definition DAGCombine.h:18
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
@ Custom
The result value requires a custom uniformity check.
Definition Uniformity.h:31
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition ValueTypes.h:90
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:330
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition ValueTypes.h:346
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
This class contains a discriminated union of information about pointers in memory operands,...