LLVM 17.0.0git
TargetLoweringBase.cpp
Go to the documentation of this file.
1//===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLoweringBase class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/BitVector.h"
14#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/Loads.h"
37#include "llvm/IR/Attributes.h"
38#include "llvm/IR/CallingConv.h"
39#include "llvm/IR/DataLayout.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/GlobalValue.h"
44#include "llvm/IR/IRBuilder.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/Type.h"
57#include <algorithm>
58#include <cassert>
59#include <cstdint>
60#include <cstring>
61#include <iterator>
62#include <string>
63#include <tuple>
64#include <utility>
65
66using namespace llvm;
67
69 "jump-is-expensive", cl::init(false),
70 cl::desc("Do not create extra branches to split comparison logic."),
72
74 ("min-jump-table-entries", cl::init(4), cl::Hidden,
75 cl::desc("Set minimum number of entries to use a jump table."));
76
78 ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden,
79 cl::desc("Set maximum size of jump tables."));
80
81/// Minimum jump table density for normal functions.
83 JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
84 cl::desc("Minimum density for building a jump table in "
85 "a normal function"));
86
87/// Minimum jump table density for -Os or -Oz functions.
89 "optsize-jump-table-density", cl::init(40), cl::Hidden,
90 cl::desc("Minimum density for building a jump table in "
91 "an optsize function"));
92
93// FIXME: This option is only to test if the strict fp operation processed
94// correctly by preventing mutating strict fp operation to normal fp operation
95// during development. When the backend supports strict float operation, this
96// option will be meaningless.
97static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation",
98 cl::desc("Don't mutate strict-float node to a legalize node"),
99 cl::init(false), cl::Hidden);
100
101static bool darwinHasSinCos(const Triple &TT) {
102 assert(TT.isOSDarwin() && "should be called with darwin triple");
103 // Don't bother with 32 bit x86.
104 if (TT.getArch() == Triple::x86)
105 return false;
106 // Macos < 10.9 has no sincos_stret.
107 if (TT.isMacOSX())
108 return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit();
109 // iOS < 7.0 has no sincos_stret.
110 if (TT.isiOS())
111 return !TT.isOSVersionLT(7, 0);
112 // Any other darwin such as WatchOS/TvOS is new enough.
113 return true;
114}
115
116void TargetLoweringBase::InitLibcalls(const Triple &TT) {
117#define HANDLE_LIBCALL(code, name) \
118 setLibcallName(RTLIB::code, name);
119#include "llvm/IR/RuntimeLibcalls.def"
120#undef HANDLE_LIBCALL
121 // Initialize calling conventions to their default.
122 for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
124
125 // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
126 if (TT.isPPC()) {
127 setLibcallName(RTLIB::ADD_F128, "__addkf3");
128 setLibcallName(RTLIB::SUB_F128, "__subkf3");
129 setLibcallName(RTLIB::MUL_F128, "__mulkf3");
130 setLibcallName(RTLIB::DIV_F128, "__divkf3");
131 setLibcallName(RTLIB::POWI_F128, "__powikf2");
132 setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
133 setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
134 setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
135 setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
136 setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
137 setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
138 setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
139 setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
140 setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
141 setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
142 setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
143 setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
144 setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
145 setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
146 setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
147 setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
148 setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
149 setLibcallName(RTLIB::UNE_F128, "__nekf2");
150 setLibcallName(RTLIB::OGE_F128, "__gekf2");
151 setLibcallName(RTLIB::OLT_F128, "__ltkf2");
152 setLibcallName(RTLIB::OLE_F128, "__lekf2");
153 setLibcallName(RTLIB::OGT_F128, "__gtkf2");
154 setLibcallName(RTLIB::UO_F128, "__unordkf2");
155 }
156
157 // A few names are different on particular architectures or environments.
158 if (TT.isOSDarwin()) {
159 // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
160 // of the gnueabi-style __gnu_*_ieee.
161 // FIXME: What about other targets?
162 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
163 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
164
165 // Some darwins have an optimized __bzero/bzero function.
166 switch (TT.getArch()) {
167 case Triple::x86:
168 case Triple::x86_64:
169 if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
170 setLibcallName(RTLIB::BZERO, "__bzero");
171 break;
172 case Triple::aarch64:
174 setLibcallName(RTLIB::BZERO, "bzero");
175 break;
176 default:
177 break;
178 }
179
180 if (darwinHasSinCos(TT)) {
181 setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
182 setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
183 if (TT.isWatchABI()) {
184 setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
186 setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
188 }
189 }
190 } else {
191 setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
192 setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
193 }
194
195 if (TT.isGNUEnvironment() || TT.isOSFuchsia() ||
196 (TT.isAndroid() && !TT.isAndroidVersionLT(9))) {
197 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
198 setLibcallName(RTLIB::SINCOS_F64, "sincos");
199 setLibcallName(RTLIB::SINCOS_F80, "sincosl");
200 setLibcallName(RTLIB::SINCOS_F128, "sincosl");
201 setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
202 }
203
204 if (TT.isPS()) {
205 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
206 setLibcallName(RTLIB::SINCOS_F64, "sincos");
207 }
208
209 if (TT.isOSOpenBSD()) {
210 setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
211 }
212}
213
214/// GetFPLibCall - Helper to return the right libcall for the given floating
215/// point type, or UNKNOWN_LIBCALL if there is none.
217 RTLIB::Libcall Call_F32,
218 RTLIB::Libcall Call_F64,
219 RTLIB::Libcall Call_F80,
220 RTLIB::Libcall Call_F128,
221 RTLIB::Libcall Call_PPCF128) {
222 return
223 VT == MVT::f32 ? Call_F32 :
224 VT == MVT::f64 ? Call_F64 :
225 VT == MVT::f80 ? Call_F80 :
226 VT == MVT::f128 ? Call_F128 :
227 VT == MVT::ppcf128 ? Call_PPCF128 :
228 RTLIB::UNKNOWN_LIBCALL;
229}
230
231/// getFPEXT - Return the FPEXT_*_* value for the given types, or
232/// UNKNOWN_LIBCALL if there is none.
234 if (OpVT == MVT::f16) {
235 if (RetVT == MVT::f32)
236 return FPEXT_F16_F32;
237 if (RetVT == MVT::f64)
238 return FPEXT_F16_F64;
239 if (RetVT == MVT::f80)
240 return FPEXT_F16_F80;
241 if (RetVT == MVT::f128)
242 return FPEXT_F16_F128;
243 } else if (OpVT == MVT::f32) {
244 if (RetVT == MVT::f64)
245 return FPEXT_F32_F64;
246 if (RetVT == MVT::f128)
247 return FPEXT_F32_F128;
248 if (RetVT == MVT::ppcf128)
249 return FPEXT_F32_PPCF128;
250 } else if (OpVT == MVT::f64) {
251 if (RetVT == MVT::f128)
252 return FPEXT_F64_F128;
253 else if (RetVT == MVT::ppcf128)
254 return FPEXT_F64_PPCF128;
255 } else if (OpVT == MVT::f80) {
256 if (RetVT == MVT::f128)
257 return FPEXT_F80_F128;
258 }
259
260 return UNKNOWN_LIBCALL;
261}
262
263/// getFPROUND - Return the FPROUND_*_* value for the given types, or
264/// UNKNOWN_LIBCALL if there is none.
266 if (RetVT == MVT::f16) {
267 if (OpVT == MVT::f32)
268 return FPROUND_F32_F16;
269 if (OpVT == MVT::f64)
270 return FPROUND_F64_F16;
271 if (OpVT == MVT::f80)
272 return FPROUND_F80_F16;
273 if (OpVT == MVT::f128)
274 return FPROUND_F128_F16;
275 if (OpVT == MVT::ppcf128)
276 return FPROUND_PPCF128_F16;
277 } else if (RetVT == MVT::bf16) {
278 if (OpVT == MVT::f32)
279 return FPROUND_F32_BF16;
280 if (OpVT == MVT::f64)
281 return FPROUND_F64_BF16;
282 } else if (RetVT == MVT::f32) {
283 if (OpVT == MVT::f64)
284 return FPROUND_F64_F32;
285 if (OpVT == MVT::f80)
286 return FPROUND_F80_F32;
287 if (OpVT == MVT::f128)
288 return FPROUND_F128_F32;
289 if (OpVT == MVT::ppcf128)
290 return FPROUND_PPCF128_F32;
291 } else if (RetVT == MVT::f64) {
292 if (OpVT == MVT::f80)
293 return FPROUND_F80_F64;
294 if (OpVT == MVT::f128)
295 return FPROUND_F128_F64;
296 if (OpVT == MVT::ppcf128)
297 return FPROUND_PPCF128_F64;
298 } else if (RetVT == MVT::f80) {
299 if (OpVT == MVT::f128)
300 return FPROUND_F128_F80;
301 }
302
303 return UNKNOWN_LIBCALL;
304}
305
306/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
307/// UNKNOWN_LIBCALL if there is none.
309 if (OpVT == MVT::f16) {
310 if (RetVT == MVT::i32)
311 return FPTOSINT_F16_I32;
312 if (RetVT == MVT::i64)
313 return FPTOSINT_F16_I64;
314 if (RetVT == MVT::i128)
315 return FPTOSINT_F16_I128;
316 } else if (OpVT == MVT::f32) {
317 if (RetVT == MVT::i32)
318 return FPTOSINT_F32_I32;
319 if (RetVT == MVT::i64)
320 return FPTOSINT_F32_I64;
321 if (RetVT == MVT::i128)
322 return FPTOSINT_F32_I128;
323 } else if (OpVT == MVT::f64) {
324 if (RetVT == MVT::i32)
325 return FPTOSINT_F64_I32;
326 if (RetVT == MVT::i64)
327 return FPTOSINT_F64_I64;
328 if (RetVT == MVT::i128)
329 return FPTOSINT_F64_I128;
330 } else if (OpVT == MVT::f80) {
331 if (RetVT == MVT::i32)
332 return FPTOSINT_F80_I32;
333 if (RetVT == MVT::i64)
334 return FPTOSINT_F80_I64;
335 if (RetVT == MVT::i128)
336 return FPTOSINT_F80_I128;
337 } else if (OpVT == MVT::f128) {
338 if (RetVT == MVT::i32)
339 return FPTOSINT_F128_I32;
340 if (RetVT == MVT::i64)
341 return FPTOSINT_F128_I64;
342 if (RetVT == MVT::i128)
343 return FPTOSINT_F128_I128;
344 } else if (OpVT == MVT::ppcf128) {
345 if (RetVT == MVT::i32)
346 return FPTOSINT_PPCF128_I32;
347 if (RetVT == MVT::i64)
348 return FPTOSINT_PPCF128_I64;
349 if (RetVT == MVT::i128)
350 return FPTOSINT_PPCF128_I128;
351 }
352 return UNKNOWN_LIBCALL;
353}
354
355/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
356/// UNKNOWN_LIBCALL if there is none.
358 if (OpVT == MVT::f16) {
359 if (RetVT == MVT::i32)
360 return FPTOUINT_F16_I32;
361 if (RetVT == MVT::i64)
362 return FPTOUINT_F16_I64;
363 if (RetVT == MVT::i128)
364 return FPTOUINT_F16_I128;
365 } else if (OpVT == MVT::f32) {
366 if (RetVT == MVT::i32)
367 return FPTOUINT_F32_I32;
368 if (RetVT == MVT::i64)
369 return FPTOUINT_F32_I64;
370 if (RetVT == MVT::i128)
371 return FPTOUINT_F32_I128;
372 } else if (OpVT == MVT::f64) {
373 if (RetVT == MVT::i32)
374 return FPTOUINT_F64_I32;
375 if (RetVT == MVT::i64)
376 return FPTOUINT_F64_I64;
377 if (RetVT == MVT::i128)
378 return FPTOUINT_F64_I128;
379 } else if (OpVT == MVT::f80) {
380 if (RetVT == MVT::i32)
381 return FPTOUINT_F80_I32;
382 if (RetVT == MVT::i64)
383 return FPTOUINT_F80_I64;
384 if (RetVT == MVT::i128)
385 return FPTOUINT_F80_I128;
386 } else if (OpVT == MVT::f128) {
387 if (RetVT == MVT::i32)
388 return FPTOUINT_F128_I32;
389 if (RetVT == MVT::i64)
390 return FPTOUINT_F128_I64;
391 if (RetVT == MVT::i128)
392 return FPTOUINT_F128_I128;
393 } else if (OpVT == MVT::ppcf128) {
394 if (RetVT == MVT::i32)
395 return FPTOUINT_PPCF128_I32;
396 if (RetVT == MVT::i64)
397 return FPTOUINT_PPCF128_I64;
398 if (RetVT == MVT::i128)
399 return FPTOUINT_PPCF128_I128;
400 }
401 return UNKNOWN_LIBCALL;
402}
403
404/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
405/// UNKNOWN_LIBCALL if there is none.
407 if (OpVT == MVT::i32) {
408 if (RetVT == MVT::f16)
409 return SINTTOFP_I32_F16;
410 if (RetVT == MVT::f32)
411 return SINTTOFP_I32_F32;
412 if (RetVT == MVT::f64)
413 return SINTTOFP_I32_F64;
414 if (RetVT == MVT::f80)
415 return SINTTOFP_I32_F80;
416 if (RetVT == MVT::f128)
417 return SINTTOFP_I32_F128;
418 if (RetVT == MVT::ppcf128)
419 return SINTTOFP_I32_PPCF128;
420 } else if (OpVT == MVT::i64) {
421 if (RetVT == MVT::f16)
422 return SINTTOFP_I64_F16;
423 if (RetVT == MVT::f32)
424 return SINTTOFP_I64_F32;
425 if (RetVT == MVT::f64)
426 return SINTTOFP_I64_F64;
427 if (RetVT == MVT::f80)
428 return SINTTOFP_I64_F80;
429 if (RetVT == MVT::f128)
430 return SINTTOFP_I64_F128;
431 if (RetVT == MVT::ppcf128)
432 return SINTTOFP_I64_PPCF128;
433 } else if (OpVT == MVT::i128) {
434 if (RetVT == MVT::f16)
435 return SINTTOFP_I128_F16;
436 if (RetVT == MVT::f32)
437 return SINTTOFP_I128_F32;
438 if (RetVT == MVT::f64)
439 return SINTTOFP_I128_F64;
440 if (RetVT == MVT::f80)
441 return SINTTOFP_I128_F80;
442 if (RetVT == MVT::f128)
443 return SINTTOFP_I128_F128;
444 if (RetVT == MVT::ppcf128)
445 return SINTTOFP_I128_PPCF128;
446 }
447 return UNKNOWN_LIBCALL;
448}
449
450/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
451/// UNKNOWN_LIBCALL if there is none.
453 if (OpVT == MVT::i32) {
454 if (RetVT == MVT::f16)
455 return UINTTOFP_I32_F16;
456 if (RetVT == MVT::f32)
457 return UINTTOFP_I32_F32;
458 if (RetVT == MVT::f64)
459 return UINTTOFP_I32_F64;
460 if (RetVT == MVT::f80)
461 return UINTTOFP_I32_F80;
462 if (RetVT == MVT::f128)
463 return UINTTOFP_I32_F128;
464 if (RetVT == MVT::ppcf128)
465 return UINTTOFP_I32_PPCF128;
466 } else if (OpVT == MVT::i64) {
467 if (RetVT == MVT::f16)
468 return UINTTOFP_I64_F16;
469 if (RetVT == MVT::f32)
470 return UINTTOFP_I64_F32;
471 if (RetVT == MVT::f64)
472 return UINTTOFP_I64_F64;
473 if (RetVT == MVT::f80)
474 return UINTTOFP_I64_F80;
475 if (RetVT == MVT::f128)
476 return UINTTOFP_I64_F128;
477 if (RetVT == MVT::ppcf128)
478 return UINTTOFP_I64_PPCF128;
479 } else if (OpVT == MVT::i128) {
480 if (RetVT == MVT::f16)
481 return UINTTOFP_I128_F16;
482 if (RetVT == MVT::f32)
483 return UINTTOFP_I128_F32;
484 if (RetVT == MVT::f64)
485 return UINTTOFP_I128_F64;
486 if (RetVT == MVT::f80)
487 return UINTTOFP_I128_F80;
488 if (RetVT == MVT::f128)
489 return UINTTOFP_I128_F128;
490 if (RetVT == MVT::ppcf128)
491 return UINTTOFP_I128_PPCF128;
492 }
493 return UNKNOWN_LIBCALL;
494}
495
497 return getFPLibCall(RetVT, POWI_F32, POWI_F64, POWI_F80, POWI_F128,
498 POWI_PPCF128);
499}
500
502 MVT VT) {
503 unsigned ModeN, ModelN;
504 switch (VT.SimpleTy) {
505 case MVT::i8:
506 ModeN = 0;
507 break;
508 case MVT::i16:
509 ModeN = 1;
510 break;
511 case MVT::i32:
512 ModeN = 2;
513 break;
514 case MVT::i64:
515 ModeN = 3;
516 break;
517 case MVT::i128:
518 ModeN = 4;
519 break;
520 default:
521 return UNKNOWN_LIBCALL;
522 }
523
524 switch (Order) {
526 ModelN = 0;
527 break;
529 ModelN = 1;
530 break;
532 ModelN = 2;
533 break;
536 ModelN = 3;
537 break;
538 default:
539 return UNKNOWN_LIBCALL;
540 }
541
542#define LCALLS(A, B) \
543 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
544#define LCALL5(A) \
545 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
546 switch (Opc) {
548 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};
549 return LC[ModeN][ModelN];
550 }
551 case ISD::ATOMIC_SWAP: {
552 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};
553 return LC[ModeN][ModelN];
554 }
556 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};
557 return LC[ModeN][ModelN];
558 }
559 case ISD::ATOMIC_LOAD_OR: {
560 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};
561 return LC[ModeN][ModelN];
562 }
564 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};
565 return LC[ModeN][ModelN];
566 }
568 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};
569 return LC[ModeN][ModelN];
570 }
571 default:
572 return UNKNOWN_LIBCALL;
573 }
574#undef LCALLS
575#undef LCALL5
576}
577
579#define OP_TO_LIBCALL(Name, Enum) \
580 case Name: \
581 switch (VT.SimpleTy) { \
582 default: \
583 return UNKNOWN_LIBCALL; \
584 case MVT::i8: \
585 return Enum##_1; \
586 case MVT::i16: \
587 return Enum##_2; \
588 case MVT::i32: \
589 return Enum##_4; \
590 case MVT::i64: \
591 return Enum##_8; \
592 case MVT::i128: \
593 return Enum##_16; \
594 }
595
596 switch (Opc) {
597 OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET)
598 OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP)
599 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD)
600 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB)
601 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND)
602 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR)
603 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR)
604 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND)
605 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX)
606 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX)
607 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN)
608 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN)
609 }
610
611#undef OP_TO_LIBCALL
612
613 return UNKNOWN_LIBCALL;
614}
615
617 switch (ElementSize) {
618 case 1:
619 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1;
620 case 2:
621 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2;
622 case 4:
623 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4;
624 case 8:
625 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8;
626 case 16:
627 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16;
628 default:
629 return UNKNOWN_LIBCALL;
630 }
631}
632
634 switch (ElementSize) {
635 case 1:
636 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;
637 case 2:
638 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;
639 case 4:
640 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;
641 case 8:
642 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;
643 case 16:
644 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;
645 default:
646 return UNKNOWN_LIBCALL;
647 }
648}
649
651 switch (ElementSize) {
652 case 1:
653 return MEMSET_ELEMENT_UNORDERED_ATOMIC_1;
654 case 2:
655 return MEMSET_ELEMENT_UNORDERED_ATOMIC_2;
656 case 4:
657 return MEMSET_ELEMENT_UNORDERED_ATOMIC_4;
658 case 8:
659 return MEMSET_ELEMENT_UNORDERED_ATOMIC_8;
660 case 16:
661 return MEMSET_ELEMENT_UNORDERED_ATOMIC_16;
662 default:
663 return UNKNOWN_LIBCALL;
664 }
665}
666
667/// InitCmpLibcallCCs - Set default comparison libcall CC.
669 std::fill(CCs, CCs + RTLIB::UNKNOWN_LIBCALL, ISD::SETCC_INVALID);
670 CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
671 CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
672 CCs[RTLIB::OEQ_F128] = ISD::SETEQ;
673 CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
674 CCs[RTLIB::UNE_F32] = ISD::SETNE;
675 CCs[RTLIB::UNE_F64] = ISD::SETNE;
676 CCs[RTLIB::UNE_F128] = ISD::SETNE;
677 CCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
678 CCs[RTLIB::OGE_F32] = ISD::SETGE;
679 CCs[RTLIB::OGE_F64] = ISD::SETGE;
680 CCs[RTLIB::OGE_F128] = ISD::SETGE;
681 CCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
682 CCs[RTLIB::OLT_F32] = ISD::SETLT;
683 CCs[RTLIB::OLT_F64] = ISD::SETLT;
684 CCs[RTLIB::OLT_F128] = ISD::SETLT;
685 CCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
686 CCs[RTLIB::OLE_F32] = ISD::SETLE;
687 CCs[RTLIB::OLE_F64] = ISD::SETLE;
688 CCs[RTLIB::OLE_F128] = ISD::SETLE;
689 CCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
690 CCs[RTLIB::OGT_F32] = ISD::SETGT;
691 CCs[RTLIB::OGT_F64] = ISD::SETGT;
692 CCs[RTLIB::OGT_F128] = ISD::SETGT;
693 CCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
694 CCs[RTLIB::UO_F32] = ISD::SETNE;
695 CCs[RTLIB::UO_F64] = ISD::SETNE;
696 CCs[RTLIB::UO_F128] = ISD::SETNE;
697 CCs[RTLIB::UO_PPCF128] = ISD::SETNE;
698}
699
700/// NOTE: The TargetMachine owns TLOF.
702 initActions();
703
704 // Perform these initializations only once.
710 HasMultipleConditionRegisters = false;
711 HasExtractBitsInsn = false;
712 JumpIsExpensive = JumpIsExpensiveOverride;
714 EnableExtLdPromotion = false;
715 StackPointerRegisterToSaveRestore = 0;
716 BooleanContents = UndefinedBooleanContent;
717 BooleanFloatContents = UndefinedBooleanContent;
718 BooleanVectorContents = UndefinedBooleanContent;
719 SchedPreferenceInfo = Sched::ILP;
722 MaxBytesForAlignment = 0;
723 // TODO: the default will be switched to 0 in the next commit, along
724 // with the Target-specific changes necessary.
725 MaxAtomicSizeInBitsSupported = 1024;
726
727 // Assume that even with libcalls, no target supports wider than 128 bit
728 // division.
729 MaxDivRemBitWidthSupported = 128;
730
731 MaxLargeFPConvertBitWidthSupported = llvm::IntegerType::MAX_INT_BITS;
732
733 MinCmpXchgSizeInBits = 0;
734 SupportsUnalignedAtomics = false;
735
736 std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
737
738 InitLibcalls(TM.getTargetTriple());
739 InitCmpLibcallCCs(CmpLibcallCCs);
740}
741
743 // All operations default to being supported.
744 memset(OpActions, 0, sizeof(OpActions));
745 memset(LoadExtActions, 0, sizeof(LoadExtActions));
746 memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
747 memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
748 memset(CondCodeActions, 0, sizeof(CondCodeActions));
749 std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
750 std::fill(std::begin(TargetDAGCombineArray),
751 std::end(TargetDAGCombineArray), 0);
752
753 // We're somewhat special casing MVT::i2 and MVT::i4. Ideally we want to
754 // remove this and targets should individually set these types if not legal.
757 for (MVT VT : {MVT::i2, MVT::i4})
758 OpActions[(unsigned)VT.SimpleTy][NT] = Expand;
759 }
760 for (MVT AVT : MVT::all_valuetypes()) {
761 for (MVT VT : {MVT::i2, MVT::i4, MVT::v128i2, MVT::v64i4}) {
762 setTruncStoreAction(AVT, VT, Expand);
765 }
766 }
767 for (unsigned IM = (unsigned)ISD::PRE_INC;
768 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
769 for (MVT VT : {MVT::i2, MVT::i4}) {
774 }
775 }
776
777 for (MVT VT : MVT::fp_valuetypes()) {
778 MVT IntVT = MVT::getIntegerVT(VT.getFixedSizeInBits());
779 if (IntVT.isValid()) {
782 }
783 }
784
785 // Set default actions for various operations.
786 for (MVT VT : MVT::all_valuetypes()) {
787 // Default all indexed load / store to expand.
788 for (unsigned IM = (unsigned)ISD::PRE_INC;
789 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
794 }
795
796 // Most backends expect to see the node which just returns the value loaded.
798
799 // These operations default to expand.
817 VT, Expand);
818
819 // Overflow operations default to expand
822 VT, Expand);
823
824 // ADDCARRY operations default to expand
827 VT, Expand);
828
829 // ADDC/ADDE/SUBC/SUBE default to expand.
831 Expand);
832
833 // Halving adds
836 Expand);
837
838 // Absolute difference
840
841 // These default to Expand so they will be expanded to CTLZ/CTTZ by default.
843 Expand);
844
846
847 // These library functions default to expand.
849
850 // These operations default to expand for vector types.
851 if (VT.isVector())
856 VT, Expand);
857
858 // Constrained floating-point operations default to expand.
859#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
860 setOperationAction(ISD::STRICT_##DAGN, VT, Expand);
861#include "llvm/IR/ConstrainedOps.def"
862
863 // For most targets @llvm.get.dynamic.area.offset just returns 0.
865
866 // Vector reduction default to expand.
873 VT, Expand);
874
875 // Named vector shuffles default to expand.
877
878 // VP_SREM/UREM default to expand.
879 // TODO: Expand all VP intrinsics.
880 setOperationAction(ISD::VP_SREM, VT, Expand);
881 setOperationAction(ISD::VP_UREM, VT, Expand);
882 }
883
884 // Most targets ignore the @llvm.prefetch intrinsic.
886
887 // Most targets also ignore the @llvm.readcyclecounter intrinsic.
889
890 // ConstantFP nodes default to expand. Targets can either change this to
891 // Legal, in which case all fp constants are legal, or use isFPImmLegal()
892 // to optimize expansions for certain constants.
895 Expand);
896
897 // These library functions default to expand.
903
904 // Default ISD::TRAP to expand (which turns it into abort).
906
907 // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
908 // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
910
912}
913
915 EVT) const {
916 return MVT::getIntegerVT(DL.getPointerSizeInBits(0));
917}
918
920 bool LegalTypes) const {
921 assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
922 if (LHSTy.isVector())
923 return LHSTy;
924 MVT ShiftVT =
925 LegalTypes ? getScalarShiftAmountTy(DL, LHSTy) : getPointerTy(DL);
926 // If any possible shift value won't fit in the prefered type, just use
927 // something safe. Assume it will be legalized when the shift is expanded.
928 if (ShiftVT.getSizeInBits() < Log2_32_Ceil(LHSTy.getSizeInBits()))
929 ShiftVT = MVT::i32;
930 assert(ShiftVT.getSizeInBits() >= Log2_32_Ceil(LHSTy.getSizeInBits()) &&
931 "ShiftVT is still too small!");
932 return ShiftVT;
933}
934
935bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
936 assert(isTypeLegal(VT));
937 switch (Op) {
938 default:
939 return false;
940 case ISD::SDIV:
941 case ISD::UDIV:
942 case ISD::SREM:
943 case ISD::UREM:
944 return true;
945 }
946}
947
949 unsigned DestAS) const {
950 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
951}
952
954 // If the command-line option was specified, ignore this request.
955 if (!JumpIsExpensiveOverride.getNumOccurrences())
956 JumpIsExpensive = isExpensive;
957}
958
961 // If this is a simple type, use the ComputeRegisterProp mechanism.
962 if (VT.isSimple()) {
963 MVT SVT = VT.getSimpleVT();
964 assert((unsigned)SVT.SimpleTy < std::size(TransformToType));
965 MVT NVT = TransformToType[SVT.SimpleTy];
966 LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);
967
968 assert((LA == TypeLegal || LA == TypeSoftenFloat ||
969 LA == TypeSoftPromoteHalf ||
970 (NVT.isVector() ||
971 ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger)) &&
972 "Promote may not follow Expand or Promote");
973
974 if (LA == TypeSplitVector)
975 return LegalizeKind(LA, EVT(SVT).getHalfNumVectorElementsVT(Context));
976 if (LA == TypeScalarizeVector)
977 return LegalizeKind(LA, SVT.getVectorElementType());
978 return LegalizeKind(LA, NVT);
979 }
980
981 // Handle Extended Scalar Types.
982 if (!VT.isVector()) {
983 assert(VT.isInteger() && "Float types must be simple");
984 unsigned BitSize = VT.getSizeInBits();
985 // First promote to a power-of-two size, then expand if necessary.
986 if (BitSize < 8 || !isPowerOf2_32(BitSize)) {
988 assert(NVT != VT && "Unable to round integer VT");
989 LegalizeKind NextStep = getTypeConversion(Context, NVT);
990 // Avoid multi-step promotion.
991 if (NextStep.first == TypePromoteInteger)
992 return NextStep;
993 // Return rounded integer type.
994 return LegalizeKind(TypePromoteInteger, NVT);
995 }
996
999 }
1000
1001 // Handle vector types.
1002 ElementCount NumElts = VT.getVectorElementCount();
1003 EVT EltVT = VT.getVectorElementType();
1004
1005 // Vectors with only one element are always scalarized.
1006 if (NumElts.isScalar())
1007 return LegalizeKind(TypeScalarizeVector, EltVT);
1008
1009 // Try to widen vector elements until the element type is a power of two and
1010 // promote it to a legal type later on, for example:
1011 // <3 x i8> -> <4 x i8> -> <4 x i32>
1012 if (EltVT.isInteger()) {
1013 // Vectors with a number of elements that is not a power of two are always
1014 // widened, for example <3 x i8> -> <4 x i8>.
1015 if (!VT.isPow2VectorType()) {
1016 NumElts = NumElts.coefficientNextPowerOf2();
1017 EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts);
1018 return LegalizeKind(TypeWidenVector, NVT);
1019 }
1020
1021 // Examine the element type.
1023
1024 // If type is to be expanded, split the vector.
1025 // <4 x i140> -> <2 x i140>
1026 if (LK.first == TypeExpandInteger) {
1031 }
1032
1033 // Promote the integer element types until a legal vector type is found
1034 // or until the element integer type is too big. If a legal type was not
1035 // found, fallback to the usual mechanism of widening/splitting the
1036 // vector.
1037 EVT OldEltVT = EltVT;
1038 while (true) {
1039 // Increase the bitwidth of the element to the next pow-of-two
1040 // (which is greater than 8 bits).
1041 EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits())
1043
1044 // Stop trying when getting a non-simple element type.
1045 // Note that vector elements may be greater than legal vector element
1046 // types. Example: X86 XMM registers hold 64bit element on 32bit
1047 // systems.
1048 if (!EltVT.isSimple())
1049 break;
1050
1051 // Build a new vector type and check if it is legal.
1052 MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
1053 // Found a legal promoted vector type.
1054 if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal)
1056 EVT::getVectorVT(Context, EltVT, NumElts));
1057 }
1058
1059 // Reset the type to the unexpanded type if we did not find a legal vector
1060 // type with a promoted vector element type.
1061 EltVT = OldEltVT;
1062 }
1063
1064 // Try to widen the vector until a legal type is found.
1065 // If there is no wider legal type, split the vector.
1066 while (true) {
1067 // Round up to the next power of 2.
1068 NumElts = NumElts.coefficientNextPowerOf2();
1069
1070 // If there is no simple vector type with this many elements then there
1071 // cannot be a larger legal vector type. Note that this assumes that
1072 // there are no skipped intermediate vector types in the simple types.
1073 if (!EltVT.isSimple())
1074 break;
1075 MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
1076 if (LargerVector == MVT())
1077 break;
1078
1079 // If this type is legal then widen the vector.
1080 if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal)
1081 return LegalizeKind(TypeWidenVector, LargerVector);
1082 }
1083
1084 // Widen odd vectors to next power of two.
1085 if (!VT.isPow2VectorType()) {
1086 EVT NVT = VT.getPow2VectorType(Context);
1087 return LegalizeKind(TypeWidenVector, NVT);
1088 }
1089
1092
1093 // Vectors with illegal element types are expanded.
1094 EVT NVT = EVT::getVectorVT(Context, EltVT,
1096 return LegalizeKind(TypeSplitVector, NVT);
1097}
1098
1099static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
1100 unsigned &NumIntermediates,
1101 MVT &RegisterVT,
1102 TargetLoweringBase *TLI) {
1103 // Figure out the right, legal destination reg to copy into.
1105 MVT EltTy = VT.getVectorElementType();
1106
1107 unsigned NumVectorRegs = 1;
1108
1109 // Scalable vectors cannot be scalarized, so splitting or widening is
1110 // required.
1111 if (VT.isScalableVector() && !isPowerOf2_32(EC.getKnownMinValue()))
1113 "Splitting or widening of non-power-of-2 MVTs is not implemented.");
1114
1115 // FIXME: We don't support non-power-of-2-sized vectors for now.
1116 // Ideally we could break down into LHS/RHS like LegalizeDAG does.
1117 if (!isPowerOf2_32(EC.getKnownMinValue())) {
1118 // Split EC to unit size (scalable property is preserved).
1119 NumVectorRegs = EC.getKnownMinValue();
1120 EC = ElementCount::getFixed(1);
1121 }
1122
1123 // Divide the input until we get to a supported size. This will
1124 // always end up with an EC that represent a scalar or a scalable
1125 // scalar.
1126 while (EC.getKnownMinValue() > 1 &&
1127 !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) {
1128 EC = EC.divideCoefficientBy(2);
1129 NumVectorRegs <<= 1;
1130 }
1131
1132 NumIntermediates = NumVectorRegs;
1133
1134 MVT NewVT = MVT::getVectorVT(EltTy, EC);
1135 if (!TLI->isTypeLegal(NewVT))
1136 NewVT = EltTy;
1137 IntermediateVT = NewVT;
1138
1139 unsigned LaneSizeInBits = NewVT.getScalarSizeInBits();
1140
1141 // Convert sizes such as i33 to i64.
1142 LaneSizeInBits = llvm::bit_ceil(LaneSizeInBits);
1143
1144 MVT DestVT = TLI->getRegisterType(NewVT);
1145 RegisterVT = DestVT;
1146 if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
1147 return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits());
1148
1149 // Otherwise, promotion or legal types use the same number of registers as
1150 // the vector decimated to the appropriate level.
1151 return NumVectorRegs;
1152}
1153
1154/// isLegalRC - Return true if the value types that can be represented by the
1155/// specified register class are all legal.
1157 const TargetRegisterClass &RC) const {
1158 for (const auto *I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I)
1159 if (isTypeLegal(*I))
1160 return true;
1161 return false;
1162}
1163
1164/// Replace/modify any TargetFrameIndex operands with a targte-dependent
1165/// sequence of memory operands that is recognized by PrologEpilogInserter.
1168 MachineBasicBlock *MBB) const {
1169 MachineInstr *MI = &InitialMI;
1170 MachineFunction &MF = *MI->getMF();
1171 MachineFrameInfo &MFI = MF.getFrameInfo();
1172
1173 // We're handling multiple types of operands here:
1174 // PATCHPOINT MetaArgs - live-in, read only, direct
1175 // STATEPOINT Deopt Spill - live-through, read only, indirect
1176 // STATEPOINT Deopt Alloca - live-through, read only, direct
1177 // (We're currently conservative and mark the deopt slots read/write in
1178 // practice.)
1179 // STATEPOINT GC Spill - live-through, read/write, indirect
1180 // STATEPOINT GC Alloca - live-through, read/write, direct
1181 // The live-in vs live-through is handled already (the live through ones are
1182 // all stack slots), but we need to handle the different type of stackmap
1183 // operands and memory effects here.
1184
1185 if (llvm::none_of(MI->operands(),
1186 [](MachineOperand &Operand) { return Operand.isFI(); }))
1187 return MBB;
1188
1189 MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc());
1190
1191 // Inherit previous memory operands.
1192 MIB.cloneMemRefs(*MI);
1193
1194 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1195 MachineOperand &MO = MI->getOperand(i);
1196 if (!MO.isFI()) {
1197 // Index of Def operand this Use it tied to.
1198 // Since Defs are coming before Uses, if Use is tied, then
1199 // index of Def must be smaller that index of that Use.
1200 // Also, Defs preserve their position in new MI.
1201 unsigned TiedTo = i;
1202 if (MO.isReg() && MO.isTied())
1203 TiedTo = MI->findTiedOperandIdx(i);
1204 MIB.add(MO);
1205 if (TiedTo < i)
1206 MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1);
1207 continue;
1208 }
1209
1210 // foldMemoryOperand builds a new MI after replacing a single FI operand
1211 // with the canonical set of five x86 addressing-mode operands.
1212 int FI = MO.getIndex();
1213
1214 // Add frame index operands recognized by stackmaps.cpp
1216 // indirect-mem-ref tag, size, #FI, offset.
1217 // Used for spills inserted by StatepointLowering. This codepath is not
1218 // used for patchpoints/stackmaps at all, for these spilling is done via
1219 // foldMemoryOperand callback only.
1220 assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity");
1221 MIB.addImm(StackMaps::IndirectMemRefOp);
1222 MIB.addImm(MFI.getObjectSize(FI));
1223 MIB.add(MO);
1224 MIB.addImm(0);
1225 } else {
1226 // direct-mem-ref tag, #FI, offset.
1227 // Used by patchpoint, and direct alloca arguments to statepoints
1228 MIB.addImm(StackMaps::DirectMemRefOp);
1229 MIB.add(MO);
1230 MIB.addImm(0);
1231 }
1232
1233 assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!");
1234
1235 // Add a new memory operand for this FI.
1236 assert(MFI.getObjectOffset(FI) != -1);
1237
1238 // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and
1239 // PATCHPOINT should be updated to do the same. (TODO)
1240 if (MI->getOpcode() != TargetOpcode::STATEPOINT) {
1245 MIB->addMemOperand(MF, MMO);
1246 }
1247 }
1249 MI->eraseFromParent();
1250 return MBB;
1251}
1252
1253/// findRepresentativeClass - Return the largest legal super-reg register class
1254/// of the register class for the specified type and its associated "cost".
1255// This function is in TargetLowering because it uses RegClassForVT which would
1256// need to be moved to TargetRegisterInfo and would necessitate moving
1257// isTypeLegal over as well - a massive change that would just require
1258// TargetLowering having a TargetRegisterInfo class member that it would use.
1259std::pair<const TargetRegisterClass *, uint8_t>
1261 MVT VT) const {
1262 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
1263 if (!RC)
1264 return std::make_pair(RC, 0);
1265
1266 // Compute the set of all super-register classes.
1267 BitVector SuperRegRC(TRI->getNumRegClasses());
1268 for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI)
1269 SuperRegRC.setBitsInMask(RCI.getMask());
1270
1271 // Find the first legal register class with the largest spill size.
1272 const TargetRegisterClass *BestRC = RC;
1273 for (unsigned i : SuperRegRC.set_bits()) {
1274 const TargetRegisterClass *SuperRC = TRI->getRegClass(i);
1275 // We want the largest possible spill size.
1276 if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC))
1277 continue;
1278 if (!isLegalRC(*TRI, *SuperRC))
1279 continue;
1280 BestRC = SuperRC;
1281 }
1282 return std::make_pair(BestRC, 1);
1283}
1284
1285/// computeRegisterProperties - Once all of the register classes are added,
1286/// this allows us to compute derived properties we expose.
1288 const TargetRegisterInfo *TRI) {
1290 "Too many value types for ValueTypeActions to hold!");
1291
1292 // Everything defaults to needing one register.
1293 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
1294 NumRegistersForVT[i] = 1;
1295 RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;
1296 }
1297 // ...except isVoid, which doesn't need any registers.
1298 NumRegistersForVT[MVT::isVoid] = 0;
1299
1300 // Find the largest integer register class.
1301 unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;
1302 for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg)
1303 assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
1304
1305 // Every integer value type larger than this largest register takes twice as
1306 // many registers to represent as the previous ValueType.
1307 for (unsigned ExpandedReg = LargestIntReg + 1;
1308 ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) {
1309 NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
1310 RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;
1311 TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);
1312 ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,
1314 }
1315
1316 // Inspect all of the ValueType's smaller than the largest integer
1317 // register to see which ones need promotion.
1318 unsigned LegalIntReg = LargestIntReg;
1319 for (unsigned IntReg = LargestIntReg - 1;
1320 IntReg >= (unsigned)MVT::i1; --IntReg) {
1321 MVT IVT = (MVT::SimpleValueType)IntReg;
1322 if (isTypeLegal(IVT)) {
1323 LegalIntReg = IntReg;
1324 } else {
1325 RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
1326 (MVT::SimpleValueType)LegalIntReg;
1327 ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
1328 }
1329 }
1330
1331 // ppcf128 type is really two f64's.
1332 if (!isTypeLegal(MVT::ppcf128)) {
1333 if (isTypeLegal(MVT::f64)) {
1334 NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];
1335 RegisterTypeForVT[MVT::ppcf128] = MVT::f64;
1336 TransformToType[MVT::ppcf128] = MVT::f64;
1337 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);
1338 } else {
1339 NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128];
1340 RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128];
1341 TransformToType[MVT::ppcf128] = MVT::i128;
1342 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat);
1343 }
1344 }
1345
1346 // Decide how to handle f128. If the target does not have native f128 support,
1347 // expand it to i128 and we will be generating soft float library calls.
1348 if (!isTypeLegal(MVT::f128)) {
1349 NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];
1350 RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];
1351 TransformToType[MVT::f128] = MVT::i128;
1352 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
1353 }
1354
1355 // Decide how to handle f80. If the target does not have native f80 support,
1356 // expand it to i96 and we will be generating soft float library calls.
1357 if (!isTypeLegal(MVT::f80)) {
1358 NumRegistersForVT[MVT::f80] = 3*NumRegistersForVT[MVT::i32];
1359 RegisterTypeForVT[MVT::f80] = RegisterTypeForVT[MVT::i32];
1360 TransformToType[MVT::f80] = MVT::i32;
1361 ValueTypeActions.setTypeAction(MVT::f80, TypeSoftenFloat);
1362 }
1363
1364 // Decide how to handle f64. If the target does not have native f64 support,
1365 // expand it to i64 and we will be generating soft float library calls.
1366 if (!isTypeLegal(MVT::f64)) {
1367 NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
1368 RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
1369 TransformToType[MVT::f64] = MVT::i64;
1370 ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);
1371 }
1372
1373 // Decide how to handle f32. If the target does not have native f32 support,
1374 // expand it to i32 and we will be generating soft float library calls.
1375 if (!isTypeLegal(MVT::f32)) {
1376 NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
1377 RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
1378 TransformToType[MVT::f32] = MVT::i32;
1379 ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);
1380 }
1381
1382 // Decide how to handle f16. If the target does not have native f16 support,
1383 // promote it to f32, because there are no f16 library calls (except for
1384 // conversions).
1385 if (!isTypeLegal(MVT::f16)) {
1386 // Allow targets to control how we legalize half.
1387 if (softPromoteHalfType()) {
1388 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16];
1389 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16];
1390 TransformToType[MVT::f16] = MVT::f32;
1391 ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf);
1392 } else {
1393 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];
1394 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];
1395 TransformToType[MVT::f16] = MVT::f32;
1396 ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);
1397 }
1398 }
1399
1400 // Decide how to handle bf16. If the target does not have native bf16 support,
1401 // promote it to f32, because there are no bf16 library calls (except for
1402 // converting from f32 to bf16).
1403 if (!isTypeLegal(MVT::bf16)) {
1404 NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];
1405 RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];
1406 TransformToType[MVT::bf16] = MVT::f32;
1407 ValueTypeActions.setTypeAction(MVT::bf16, TypeSoftPromoteHalf);
1408 }
1409
1410 // Loop over all of the vector value types to see which need transformations.
1411 for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
1413 MVT VT = (MVT::SimpleValueType) i;
1414 if (isTypeLegal(VT))
1415 continue;
1416
1417 MVT EltVT = VT.getVectorElementType();
1419 bool IsLegalWiderType = false;
1420 bool IsScalable = VT.isScalableVector();
1421 LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT);
1422 switch (PreferredAction) {
1423 case TypePromoteInteger: {
1424 MVT::SimpleValueType EndVT = IsScalable ?
1427 // Try to promote the elements of integer vectors. If no legal
1428 // promotion was found, fall through to the widen-vector method.
1429 for (unsigned nVT = i + 1;
1430 (MVT::SimpleValueType)nVT <= EndVT; ++nVT) {
1431 MVT SVT = (MVT::SimpleValueType) nVT;
1432 // Promote vectors of integers to vectors with the same number
1433 // of elements, with a wider element type.
1434 if (SVT.getScalarSizeInBits() > EltVT.getFixedSizeInBits() &&
1435 SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) {
1436 TransformToType[i] = SVT;
1437 RegisterTypeForVT[i] = SVT;
1438 NumRegistersForVT[i] = 1;
1439 ValueTypeActions.setTypeAction(VT, TypePromoteInteger);
1440 IsLegalWiderType = true;
1441 break;
1442 }
1443 }
1444 if (IsLegalWiderType)
1445 break;
1446 [[fallthrough]];
1447 }
1448
1449 case TypeWidenVector:
1450 if (isPowerOf2_32(EC.getKnownMinValue())) {
1451 // Try to widen the vector.
1452 for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
1453 MVT SVT = (MVT::SimpleValueType) nVT;
1454 if (SVT.getVectorElementType() == EltVT &&
1455 SVT.isScalableVector() == IsScalable &&
1457 EC.getKnownMinValue() &&
1458 isTypeLegal(SVT)) {
1459 TransformToType[i] = SVT;
1460 RegisterTypeForVT[i] = SVT;
1461 NumRegistersForVT[i] = 1;
1462 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1463 IsLegalWiderType = true;
1464 break;
1465 }
1466 }
1467 if (IsLegalWiderType)
1468 break;
1469 } else {
1470 // Only widen to the next power of 2 to keep consistency with EVT.
1471 MVT NVT = VT.getPow2VectorType();
1472 if (isTypeLegal(NVT)) {
1473 TransformToType[i] = NVT;
1474 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1475 RegisterTypeForVT[i] = NVT;
1476 NumRegistersForVT[i] = 1;
1477 break;
1478 }
1479 }
1480 [[fallthrough]];
1481
1482 case TypeSplitVector:
1483 case TypeScalarizeVector: {
1484 MVT IntermediateVT;
1485 MVT RegisterVT;
1486 unsigned NumIntermediates;
1487 unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT,
1488 NumIntermediates, RegisterVT, this);
1489 NumRegistersForVT[i] = NumRegisters;
1490 assert(NumRegistersForVT[i] == NumRegisters &&
1491 "NumRegistersForVT size cannot represent NumRegisters!");
1492 RegisterTypeForVT[i] = RegisterVT;
1493
1494 MVT NVT = VT.getPow2VectorType();
1495 if (NVT == VT) {
1496 // Type is already a power of 2. The default action is to split.
1497 TransformToType[i] = MVT::Other;
1498 if (PreferredAction == TypeScalarizeVector)
1499 ValueTypeActions.setTypeAction(VT, TypeScalarizeVector);
1500 else if (PreferredAction == TypeSplitVector)
1501 ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1502 else if (EC.getKnownMinValue() > 1)
1503 ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1504 else
1505 ValueTypeActions.setTypeAction(VT, EC.isScalable()
1508 } else {
1509 TransformToType[i] = NVT;
1510 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1511 }
1512 break;
1513 }
1514 default:
1515 llvm_unreachable("Unknown vector legalization action!");
1516 }
1517 }
1518
1519 // Determine the 'representative' register class for each value type.
1520 // An representative register class is the largest (meaning one which is
1521 // not a sub-register class / subreg register class) legal register class for
1522 // a group of value types. For example, on i386, i8, i16, and i32
1523 // representative would be GR32; while on x86_64 it's GR64.
1524 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
1525 const TargetRegisterClass* RRC;
1526 uint8_t Cost;
1528 RepRegClassForVT[i] = RRC;
1529 RepRegClassCostForVT[i] = Cost;
1530 }
1531}
1532
1534 EVT VT) const {
1535 assert(!VT.isVector() && "No default SetCC type for vectors!");
1536 return getPointerTy(DL).SimpleTy;
1537}
1538
1540 return MVT::i32; // return the default value
1541}
1542
1543/// getVectorTypeBreakdown - Vector types are broken down into some number of
1544/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
1545/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
1546/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
1547///
1548/// This method returns the number of registers needed, and the VT for each
1549/// register. It also returns the VT and quantity of the intermediate values
1550/// before they are promoted/expanded.
1552 EVT VT, EVT &IntermediateVT,
1553 unsigned &NumIntermediates,
1554 MVT &RegisterVT) const {
1555 ElementCount EltCnt = VT.getVectorElementCount();
1556
1557 // If there is a wider vector type with the same element type as this one,
1558 // or a promoted vector type that has the same number of elements which
1559 // are wider, then we should convert to that legal vector type.
1560 // This handles things like <2 x float> -> <4 x float> and
1561 // <4 x i1> -> <4 x i32>.
1563 if (!EltCnt.isScalar() &&
1564 (TA == TypeWidenVector || TA == TypePromoteInteger)) {
1565 EVT RegisterEVT = getTypeToTransformTo(Context, VT);
1566 if (isTypeLegal(RegisterEVT)) {
1567 IntermediateVT = RegisterEVT;
1568 RegisterVT = RegisterEVT.getSimpleVT();
1569 NumIntermediates = 1;
1570 return 1;
1571 }
1572 }
1573
1574 // Figure out the right, legal destination reg to copy into.
1575 EVT EltTy = VT.getVectorElementType();
1576
1577 unsigned NumVectorRegs = 1;
1578
1579 // Scalable vectors cannot be scalarized, so handle the legalisation of the
1580 // types like done elsewhere in SelectionDAG.
1581 if (EltCnt.isScalable()) {
1582 LegalizeKind LK;
1583 EVT PartVT = VT;
1584 do {
1585 // Iterate until we've found a legal (part) type to hold VT.
1586 LK = getTypeConversion(Context, PartVT);
1587 PartVT = LK.second;
1588 } while (LK.first != TypeLegal);
1589
1590 if (!PartVT.isVector()) {
1592 "Don't know how to legalize this scalable vector type");
1593 }
1594
1595 NumIntermediates =
1598 IntermediateVT = PartVT;
1599 RegisterVT = getRegisterType(Context, IntermediateVT);
1600 return NumIntermediates;
1601 }
1602
1603 // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally
1604 // we could break down into LHS/RHS like LegalizeDAG does.
1605 if (!isPowerOf2_32(EltCnt.getKnownMinValue())) {
1606 NumVectorRegs = EltCnt.getKnownMinValue();
1607 EltCnt = ElementCount::getFixed(1);
1608 }
1609
1610 // Divide the input until we get to a supported size. This will always
1611 // end with a scalar if the target doesn't support vectors.
1612 while (EltCnt.getKnownMinValue() > 1 &&
1613 !isTypeLegal(EVT::getVectorVT(Context, EltTy, EltCnt))) {
1614 EltCnt = EltCnt.divideCoefficientBy(2);
1615 NumVectorRegs <<= 1;
1616 }
1617
1618 NumIntermediates = NumVectorRegs;
1619
1620 EVT NewVT = EVT::getVectorVT(Context, EltTy, EltCnt);
1621 if (!isTypeLegal(NewVT))
1622 NewVT = EltTy;
1623 IntermediateVT = NewVT;
1624
1625 MVT DestVT = getRegisterType(Context, NewVT);
1626 RegisterVT = DestVT;
1627
1628 if (EVT(DestVT).bitsLT(NewVT)) { // Value is expanded, e.g. i64 -> i16.
1629 TypeSize NewVTSize = NewVT.getSizeInBits();
1630 // Convert sizes such as i33 to i64.
1631 if (!llvm::has_single_bit<uint32_t>(NewVTSize.getKnownMinValue()))
1632 NewVTSize = NewVTSize.coefficientNextPowerOf2();
1633 return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
1634 }
1635
1636 // Otherwise, promotion or legal types use the same number of registers as
1637 // the vector decimated to the appropriate level.
1638 return NumVectorRegs;
1639}
1640
1642 uint64_t NumCases,
1643 uint64_t Range,
1644 ProfileSummaryInfo *PSI,
1645 BlockFrequencyInfo *BFI) const {
1646 // FIXME: This function check the maximum table size and density, but the
1647 // minimum size is not checked. It would be nice if the minimum size is
1648 // also combined within this function. Currently, the minimum size check is
1649 // performed in findJumpTable() in SelectionDAGBuiler and
1650 // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
1651 const bool OptForSize =
1652 SI->getParent()->getParent()->hasOptSize() ||
1653 llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI);
1654 const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);
1655 const unsigned MaxJumpTableSize = getMaximumJumpTableSize();
1656
1657 // Check whether the number of cases is small enough and
1658 // the range is dense enough for a jump table.
1659 return (OptForSize || Range <= MaxJumpTableSize) &&
1660 (NumCases * 100 >= Range * MinDensity);
1661}
1662
1664 EVT ConditionVT) const {
1665 return getRegisterType(Context, ConditionVT);
1666}
1667
1668/// Get the EVTs and ArgFlags collections that represent the legalized return
1669/// type of the given function. This does not require a DAG or a return value,
1670/// and is suitable for use before any DAGs for the function are constructed.
1671/// TODO: Move this out of TargetLowering.cpp.
1673 AttributeList attr,
1675 const TargetLowering &TLI, const DataLayout &DL) {
1676 SmallVector<EVT, 4> ValueVTs;
1677 ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
1678 unsigned NumValues = ValueVTs.size();
1679 if (NumValues == 0) return;
1680
1681 for (unsigned j = 0, f = NumValues; j != f; ++j) {
1682 EVT VT = ValueVTs[j];
1683 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1684
1685 if (attr.hasRetAttr(Attribute::SExt))
1686 ExtendKind = ISD::SIGN_EXTEND;
1687 else if (attr.hasRetAttr(Attribute::ZExt))
1688 ExtendKind = ISD::ZERO_EXTEND;
1689
1690 // FIXME: C calling convention requires the return type to be promoted to
1691 // at least 32-bit. But this is not necessary for non-C calling
1692 // conventions. The frontend should mark functions whose return values
1693 // require promoting with signext or zeroext attributes.
1694 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) {
1695 MVT MinVT = TLI.getRegisterType(MVT::i32);
1696 if (VT.bitsLT(MinVT))
1697 VT = MinVT;
1698 }
1699
1700 unsigned NumParts =
1701 TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT);
1702 MVT PartVT =
1703 TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT);
1704
1705 // 'inreg' on function refers to return value
1707 if (attr.hasRetAttr(Attribute::InReg))
1708 Flags.setInReg();
1709
1710 // Propagate extension type if any
1711 if (attr.hasRetAttr(Attribute::SExt))
1712 Flags.setSExt();
1713 else if (attr.hasRetAttr(Attribute::ZExt))
1714 Flags.setZExt();
1715
1716 for (unsigned i = 0; i < NumParts; ++i)
1717 Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0));
1718 }
1719}
1720
1721/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1722/// function arguments in the caller parameter area. This is the actual
1723/// alignment, not its logarithm.
1725 const DataLayout &DL) const {
1726 return DL.getABITypeAlign(Ty).value();
1727}
1728
1730 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
1731 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
1732 // Check if the specified alignment is sufficient based on the data layout.
1733 // TODO: While using the data layout works in practice, a better solution
1734 // would be to implement this check directly (make this a virtual function).
1735 // For example, the ABI alignment may change based on software platform while
1736 // this function should only be affected by hardware implementation.
1737 Type *Ty = VT.getTypeForEVT(Context);
1738 if (VT.isZeroSized() || Alignment >= DL.getABITypeAlign(Ty)) {
1739 // Assume that an access that meets the ABI-specified alignment is fast.
1740 if (Fast != nullptr)
1741 *Fast = 1;
1742 return true;
1743 }
1744
1745 // This is a misaligned access.
1746 return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
1747}
1748
1750 LLVMContext &Context, const DataLayout &DL, EVT VT,
1751 const MachineMemOperand &MMO, unsigned *Fast) const {
1753 MMO.getAlign(), MMO.getFlags(), Fast);
1754}
1755
1757 const DataLayout &DL, EVT VT,
1758 unsigned AddrSpace, Align Alignment,
1760 unsigned *Fast) const {
1761 return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment,
1762 Flags, Fast);
1763}
1764
1766 const DataLayout &DL, EVT VT,
1767 const MachineMemOperand &MMO,
1768 unsigned *Fast) const {
1769 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
1770 MMO.getFlags(), Fast);
1771}
1772
1774 const DataLayout &DL, LLT Ty,
1775 const MachineMemOperand &MMO,
1776 unsigned *Fast) const {
1778 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
1779 MMO.getFlags(), Fast);
1780}
1781
1782//===----------------------------------------------------------------------===//
1783// TargetTransformInfo Helpers
1784//===----------------------------------------------------------------------===//
1785
1787 enum InstructionOpcodes {
1788#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
1789#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
1790#include "llvm/IR/Instruction.def"
1791 };
1792 switch (static_cast<InstructionOpcodes>(Opcode)) {
1793 case Ret: return 0;
1794 case Br: return 0;
1795 case Switch: return 0;
1796 case IndirectBr: return 0;
1797 case Invoke: return 0;
1798 case CallBr: return 0;
1799 case Resume: return 0;
1800 case Unreachable: return 0;
1801 case CleanupRet: return 0;
1802 case CatchRet: return 0;
1803 case CatchPad: return 0;
1804 case CatchSwitch: return 0;
1805 case CleanupPad: return 0;
1806 case FNeg: return ISD::FNEG;
1807 case Add: return ISD::ADD;
1808 case FAdd: return ISD::FADD;
1809 case Sub: return ISD::SUB;
1810 case FSub: return ISD::FSUB;
1811 case Mul: return ISD::MUL;
1812 case FMul: return ISD::FMUL;
1813 case UDiv: return ISD::UDIV;
1814 case SDiv: return ISD::SDIV;
1815 case FDiv: return ISD::FDIV;
1816 case URem: return ISD::UREM;
1817 case SRem: return ISD::SREM;
1818 case FRem: return ISD::FREM;
1819 case Shl: return ISD::SHL;
1820 case LShr: return ISD::SRL;
1821 case AShr: return ISD::SRA;
1822 case And: return ISD::AND;
1823 case Or: return ISD::OR;
1824 case Xor: return ISD::XOR;
1825 case Alloca: return 0;
1826 case Load: return ISD::LOAD;
1827 case Store: return ISD::STORE;
1828 case GetElementPtr: return 0;
1829 case Fence: return 0;
1830 case AtomicCmpXchg: return 0;
1831 case AtomicRMW: return 0;
1832 case Trunc: return ISD::TRUNCATE;
1833 case ZExt: return ISD::ZERO_EXTEND;
1834 case SExt: return ISD::SIGN_EXTEND;
1835 case FPToUI: return ISD::FP_TO_UINT;
1836 case FPToSI: return ISD::FP_TO_SINT;
1837 case UIToFP: return ISD::UINT_TO_FP;
1838 case SIToFP: return ISD::SINT_TO_FP;
1839 case FPTrunc: return ISD::FP_ROUND;
1840 case FPExt: return ISD::FP_EXTEND;
1841 case PtrToInt: return ISD::BITCAST;
1842 case IntToPtr: return ISD::BITCAST;
1843 case BitCast: return ISD::BITCAST;
1844 case AddrSpaceCast: return ISD::ADDRSPACECAST;
1845 case ICmp: return ISD::SETCC;
1846 case FCmp: return ISD::SETCC;
1847 case PHI: return 0;
1848 case Call: return 0;
1849 case Select: return ISD::SELECT;
1850 case UserOp1: return 0;
1851 case UserOp2: return 0;
1852 case VAArg: return 0;
1853 case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
1854 case InsertElement: return ISD::INSERT_VECTOR_ELT;
1855 case ShuffleVector: return ISD::VECTOR_SHUFFLE;
1856 case ExtractValue: return ISD::MERGE_VALUES;
1857 case InsertValue: return ISD::MERGE_VALUES;
1858 case LandingPad: return 0;
1859 case Freeze: return ISD::FREEZE;
1860 }
1861
1862 llvm_unreachable("Unknown instruction type encountered!");
1863}
1864
1865Value *
1867 bool UseTLS) const {
1868 // compiler-rt provides a variable with a magic name. Targets that do not
1869 // link with compiler-rt may also provide such a variable.
1870 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1871 const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";
1872 auto UnsafeStackPtr =
1873 dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar));
1874
1875 Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
1876
1877 if (!UnsafeStackPtr) {
1878 auto TLSModel = UseTLS ?
1881 // The global variable is not defined yet, define it ourselves.
1882 // We use the initial-exec TLS model because we do not support the
1883 // variable living anywhere other than in the main executable.
1884 UnsafeStackPtr = new GlobalVariable(
1885 *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr,
1886 UnsafeStackPtrVar, nullptr, TLSModel);
1887 } else {
1888 // The variable exists, check its type and attributes.
1889 if (UnsafeStackPtr->getValueType() != StackPtrTy)
1890 report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type");
1891 if (UseTLS != UnsafeStackPtr->isThreadLocal())
1892 report_fatal_error(Twine(UnsafeStackPtrVar) + " must " +
1893 (UseTLS ? "" : "not ") + "be thread-local");
1894 }
1895 return UnsafeStackPtr;
1896}
1897
1898Value *
1900 if (!TM.getTargetTriple().isAndroid())
1901 return getDefaultSafeStackPointerLocation(IRB, true);
1902
1903 // Android provides a libc function to retrieve the address of the current
1904 // thread's unsafe stack pointer.
1905 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1906 Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
1907 FunctionCallee Fn = M->getOrInsertFunction("__safestack_pointer_address",
1908 StackPtrTy->getPointerTo(0));
1909 return IRB.CreateCall(Fn);
1910}
1911
1912//===----------------------------------------------------------------------===//
1913// Loop Strength Reduction hooks
1914//===----------------------------------------------------------------------===//
1915
1916/// isLegalAddressingMode - Return true if the addressing mode represented
1917/// by AM is legal for this target, for a load/store of the specified type.
1919 const AddrMode &AM, Type *Ty,
1920 unsigned AS, Instruction *I) const {
1921 // The default implementation of this implements a conservative RISCy, r+r and
1922 // r+i addr mode.
1923
1924 // Allows a sign-extended 16-bit immediate field.
1925 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
1926 return false;
1927
1928 // No global is ever allowed as a base.
1929 if (AM.BaseGV)
1930 return false;
1931
1932 // Only support r+r,
1933 switch (AM.Scale) {
1934 case 0: // "r+i" or just "i", depending on HasBaseReg.
1935 break;
1936 case 1:
1937 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
1938 return false;
1939 // Otherwise we have r+r or r+i.
1940 break;
1941 case 2:
1942 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
1943 return false;
1944 // Allow 2*r as r+r.
1945 break;
1946 default: // Don't allow n * r
1947 return false;
1948 }
1949
1950 return true;
1951}
1952
1953//===----------------------------------------------------------------------===//
1954// Stack Protector
1955//===----------------------------------------------------------------------===//
1956
1957// For OpenBSD return its special guard variable. Otherwise return nullptr,
1958// so that SelectionDAG handle SSP.
1960 if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
1961 Module &M = *IRB.GetInsertBlock()->getParent()->getParent();
1962 PointerType *PtrTy = Type::getInt8PtrTy(M.getContext());
1963 Constant *C = M.getOrInsertGlobal("__guard_local", PtrTy);
1964 if (GlobalVariable *G = dyn_cast_or_null<GlobalVariable>(C))
1965 G->setVisibility(GlobalValue::HiddenVisibility);
1966 return C;
1967 }
1968 return nullptr;
1969}
1970
1971// Currently only support "standard" __stack_chk_guard.
1972// TODO: add LOAD_STACK_GUARD support.
1974 if (!M.getNamedValue("__stack_chk_guard")) {
1975 auto *GV = new GlobalVariable(M, Type::getInt8PtrTy(M.getContext()), false,
1977 "__stack_chk_guard");
1978
1979 // FreeBSD has "__stack_chk_guard" defined externally on libc.so
1980 if (TM.getRelocationModel() == Reloc::Static &&
1983 GV->setDSOLocal(true);
1984 }
1985}
1986
1987// Currently only support "standard" __stack_chk_guard.
1988// TODO: add LOAD_STACK_GUARD support.
1990 return M.getNamedValue("__stack_chk_guard");
1991}
1992
1994 return nullptr;
1995}
1996
1999}
2000
2003}
2004
2005unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const {
2006 return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity;
2007}
2008
2010 return MaximumJumpTableSize;
2011}
2012
2015}
2016
2019}
2020
2022 if (TM.Options.LoopAlignment)
2023 return Align(TM.Options.LoopAlignment);
2024 return PrefLoopAlignment;
2025}
2026
2028 MachineBasicBlock *MBB) const {
2029 return MaxBytesForAlignment;
2030}
2031
2032//===----------------------------------------------------------------------===//
2033// Reciprocal Estimates
2034//===----------------------------------------------------------------------===//
2035
2036/// Get the reciprocal estimate attribute string for a function that will
2037/// override the target defaults.
2039 const Function &F = MF.getFunction();
2040 return F.getFnAttribute("reciprocal-estimates").getValueAsString();
2041}
2042
2043/// Construct a string for the given reciprocal operation of the given type.
2044/// This string should match the corresponding option to the front-end's
2045/// "-mrecip" flag assuming those strings have been passed through in an
2046/// attribute string. For example, "vec-divf" for a division of a vXf32.
2047static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
2048 std::string Name = VT.isVector() ? "vec-" : "";
2049
2050 Name += IsSqrt ? "sqrt" : "div";
2051
2052 // TODO: Handle other float types?
2053 if (VT.getScalarType() == MVT::f64) {
2054 Name += "d";
2055 } else if (VT.getScalarType() == MVT::f16) {
2056 Name += "h";
2057 } else {
2058 assert(VT.getScalarType() == MVT::f32 &&
2059 "Unexpected FP type for reciprocal estimate");
2060 Name += "f";
2061 }
2062
2063 return Name;
2064}
2065
2066/// Return the character position and value (a single numeric character) of a
2067/// customized refinement operation in the input string if it exists. Return
2068/// false if there is no customized refinement step count.
2069static bool parseRefinementStep(StringRef In, size_t &Position,
2070 uint8_t &Value) {
2071 const char RefStepToken = ':';
2072 Position = In.find(RefStepToken);
2073 if (Position == StringRef::npos)
2074 return false;
2075
2076 StringRef RefStepString = In.substr(Position + 1);
2077 // Allow exactly one numeric character for the additional refinement
2078 // step parameter.
2079 if (RefStepString.size() == 1) {
2080 char RefStepChar = RefStepString[0];
2081 if (isDigit(RefStepChar)) {
2082 Value = RefStepChar - '0';
2083 return true;
2084 }
2085 }
2086 report_fatal_error("Invalid refinement step for -recip.");
2087}
2088
2089/// For the input attribute string, return one of the ReciprocalEstimate enum
2090/// status values (enabled, disabled, or not specified) for this operation on
2091/// the specified data type.
2092static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
2093 if (Override.empty())
2095
2096 SmallVector<StringRef, 4> OverrideVector;
2097 Override.split(OverrideVector, ',');
2098 unsigned NumArgs = OverrideVector.size();
2099
2100 // Check if "all", "none", or "default" was specified.
2101 if (NumArgs == 1) {
2102 // Look for an optional setting of the number of refinement steps needed
2103 // for this type of reciprocal operation.
2104 size_t RefPos;
2105 uint8_t RefSteps;
2106 if (parseRefinementStep(Override, RefPos, RefSteps)) {
2107 // Split the string for further processing.
2108 Override = Override.substr(0, RefPos);
2109 }
2110
2111 // All reciprocal types are enabled.
2112 if (Override == "all")
2114
2115 // All reciprocal types are disabled.
2116 if (Override == "none")
2118
2119 // Target defaults for enablement are used.
2120 if (Override == "default")
2122 }
2123
2124 // The attribute string may omit the size suffix ('f'/'d').
2125 std::string VTName = getReciprocalOpName(IsSqrt, VT);
2126 std::string VTNameNoSize = VTName;
2127 VTNameNoSize.pop_back();
2128 static const char DisabledPrefix = '!';
2129
2130 for (StringRef RecipType : OverrideVector) {
2131 size_t RefPos;
2132 uint8_t RefSteps;
2133 if (parseRefinementStep(RecipType, RefPos, RefSteps))
2134 RecipType = RecipType.substr(0, RefPos);
2135
2136 // Ignore the disablement token for string matching.
2137 bool IsDisabled = RecipType[0] == DisabledPrefix;
2138 if (IsDisabled)
2139 RecipType = RecipType.substr(1);
2140
2141 if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
2144 }
2145
2147}
2148
2149/// For the input attribute string, return the customized refinement step count
2150/// for this operation on the specified data type. If the step count does not
2151/// exist, return the ReciprocalEstimate enum value for unspecified.
2152static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
2153 if (Override.empty())
2155
2156 SmallVector<StringRef, 4> OverrideVector;
2157 Override.split(OverrideVector, ',');
2158 unsigned NumArgs = OverrideVector.size();
2159
2160 // Check if "all", "default", or "none" was specified.
2161 if (NumArgs == 1) {
2162 // Look for an optional setting of the number of refinement steps needed
2163 // for this type of reciprocal operation.
2164 size_t RefPos;
2165 uint8_t RefSteps;
2166 if (!parseRefinementStep(Override, RefPos, RefSteps))
2168
2169 // Split the string for further processing.
2170 Override = Override.substr(0, RefPos);
2171 assert(Override != "none" &&
2172 "Disabled reciprocals, but specifed refinement steps?");
2173
2174 // If this is a general override, return the specified number of steps.
2175 if (Override == "all" || Override == "default")
2176 return RefSteps;
2177 }
2178
2179 // The attribute string may omit the size suffix ('f'/'d').
2180 std::string VTName = getReciprocalOpName(IsSqrt, VT);
2181 std::string VTNameNoSize = VTName;
2182 VTNameNoSize.pop_back();
2183
2184 for (StringRef RecipType : OverrideVector) {
2185 size_t RefPos;
2186 uint8_t RefSteps;
2187 if (!parseRefinementStep(RecipType, RefPos, RefSteps))
2188 continue;
2189
2190 RecipType = RecipType.substr(0, RefPos);
2191 if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
2192 return RefSteps;
2193 }
2194
2196}
2197
2199 MachineFunction &MF) const {
2200 return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
2201}
2202
2204 MachineFunction &MF) const {
2205 return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
2206}
2207
2209 MachineFunction &MF) const {
2210 return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
2211}
2212
2214 MachineFunction &MF) const {
2215 return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
2216}
2217
2219 EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG,
2220 const MachineMemOperand &MMO) const {
2221 // Single-element vectors are scalarized, so we should generally avoid having
2222 // any memory operations on such types, as they would get scalarized too.
2223 if (LoadVT.isFixedLengthVector() && BitcastVT.isFixedLengthVector() &&
2224 BitcastVT.getVectorNumElements() == 1)
2225 return false;
2226
2227 // Don't do if we could do an indexed load on the original type, but not on
2228 // the new one.
2229 if (!LoadVT.isSimple() || !BitcastVT.isSimple())
2230 return true;
2231
2232 MVT LoadMVT = LoadVT.getSimpleVT();
2233
2234 // Don't bother doing this if it's just going to be promoted again later, as
2235 // doing so might interfere with other combines.
2236 if (getOperationAction(ISD::LOAD, LoadMVT) == Promote &&
2237 getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())
2238 return false;
2239
2240 unsigned Fast = 0;
2241 return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,
2242 MMO, &Fast) &&
2243 Fast;
2244}
2245
2248}
2249
2251 const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC,
2252 const TargetLibraryInfo *LibInfo) const {
2254 if (LI.isVolatile())
2256
2257 if (LI.hasMetadata(LLVMContext::MD_nontemporal))
2259
2260 if (LI.hasMetadata(LLVMContext::MD_invariant_load))
2262
2264 LI.getAlign(), DL, &LI, AC,
2265 /*DT=*/nullptr, LibInfo))
2267
2268 Flags |= getTargetMMOFlags(LI);
2269 return Flags;
2270}
2271
2274 const DataLayout &DL) const {
2276
2277 if (SI.isVolatile())
2279
2280 if (SI.hasMetadata(LLVMContext::MD_nontemporal))
2282
2283 // FIXME: Not preserving dereferenceable
2285 return Flags;
2286}
2287
2290 const DataLayout &DL) const {
2292
2293 if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) {
2294 if (RMW->isVolatile())
2296 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) {
2297 if (CmpX->isVolatile())
2299 } else
2300 llvm_unreachable("not an atomic instruction");
2301
2302 // FIXME: Not preserving dereferenceable
2303 Flags |= getTargetMMOFlags(AI);
2304 return Flags;
2305}
2306
2308 Instruction *Inst,
2309 AtomicOrdering Ord) const {
2310 if (isReleaseOrStronger(Ord) && Inst->hasAtomicStore())
2311 return Builder.CreateFence(Ord);
2312 else
2313 return nullptr;
2314}
2315
2317 Instruction *Inst,
2318 AtomicOrdering Ord) const {
2319 if (isAcquireOrStronger(Ord))
2320 return Builder.CreateFence(Ord);
2321 else
2322 return nullptr;
2323}
2324
2325//===----------------------------------------------------------------------===//
2326// GlobalISel Hooks
2327//===----------------------------------------------------------------------===//
2328
2330 const TargetTransformInfo *TTI) const {
2331 auto &MF = *MI.getMF();
2332 auto &MRI = MF.getRegInfo();
2333 // Assuming a spill and reload of a value has a cost of 1 instruction each,
2334 // this helper function computes the maximum number of uses we should consider
2335 // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We
2336 // break even in terms of code size when the original MI has 2 users vs
2337 // choosing to potentially spill. Any more than 2 users we we have a net code
2338 // size increase. This doesn't take into account register pressure though.
2339 auto maxUses = [](unsigned RematCost) {
2340 // A cost of 1 means remats are basically free.
2341 if (RematCost == 1)
2342 return std::numeric_limits<unsigned>::max();
2343 if (RematCost == 2)
2344 return 2U;
2345
2346 // Remat is too expensive, only sink if there's one user.
2347 if (RematCost > 2)
2348 return 1U;
2349 llvm_unreachable("Unexpected remat cost");
2350 };
2351
2352 switch (MI.getOpcode()) {
2353 default:
2354 return false;
2355 // Constants-like instructions should be close to their users.
2356 // We don't want long live-ranges for them.
2357 case TargetOpcode::G_CONSTANT:
2358 case TargetOpcode::G_FCONSTANT:
2359 case TargetOpcode::G_FRAME_INDEX:
2360 case TargetOpcode::G_INTTOPTR:
2361 return true;
2362 case TargetOpcode::G_GLOBAL_VALUE: {
2363 unsigned RematCost = TTI->getGISelRematGlobalCost();
2364 Register Reg = MI.getOperand(0).getReg();
2365 unsigned MaxUses = maxUses(RematCost);
2366 if (MaxUses == UINT_MAX)
2367 return true; // Remats are "free" so always localize.
2368 return MRI.hasAtMostUserInstrs(Reg, MaxUses);
2369 }
2370 }
2371}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
Rewrite undef for PHI
assume Assume Builder
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
std::string Name
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
LLVMContext & Context
const char LLVMTargetMachineRef TM
static bool isDigit(const char C)
@ SI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
static bool darwinHasSinCos(const Triple &TT)
static cl::opt< bool > JumpIsExpensiveOverride("jump-is-expensive", cl::init(false), cl::desc("Do not create extra branches to split comparison logic."), cl::Hidden)
#define OP_TO_LIBCALL(Name, Enum)
static cl::opt< unsigned > MinimumJumpTableEntries("min-jump-table-entries", cl::init(4), cl::Hidden, cl::desc("Set minimum number of entries to use a jump table."))
static cl::opt< bool > DisableStrictNodeMutation("disable-strictnode-mutation", cl::desc("Don't mutate strict-float node to a legalize node"), cl::init(false), cl::Hidden)
static bool parseRefinementStep(StringRef In, size_t &Position, uint8_t &Value)
Return the character position and value (a single numeric character) of a customized refinement opera...
static cl::opt< unsigned > MaximumJumpTableSize("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden, cl::desc("Set maximum size of jump tables."))
static cl::opt< unsigned > JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden, cl::desc("Minimum density for building a jump table in " "a normal function"))
Minimum jump table density for normal functions.
static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT, TargetLoweringBase *TLI)
static std::string getReciprocalOpName(bool IsSqrt, EVT VT)
Construct a string for the given reciprocal operation of the given type.
#define LCALL5(A)
static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override)
For the input attribute string, return the customized refinement step count for this operation on the...
static void InitCmpLibcallCCs(ISD::CondCode *CCs)
InitCmpLibcallCCs - Set default comparison libcall CC.
static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override)
For the input attribute string, return one of the ReciprocalEstimate enum status values (enabled,...
static StringRef getRecipEstimateForFunc(MachineFunction &MF)
Get the reciprocal estimate attribute string for a function that will override the target defaults.
static cl::opt< unsigned > OptsizeJumpTableDensity("optsize-jump-table-density", cl::init(40), cl::Hidden, cl::desc("Minimum density for building a jump table in " "an optsize function"))
Minimum jump table density for -Os or -Oz functions.
This file describes how to lower LLVM code to machine code.
This pass exposes codegen information to IR-level passes.
@ Flags
Definition: TextStubV5.cpp:93
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:513
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:718
bool hasRetAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the return value.
Definition: Attributes.h:785
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Definition: BitVector.h:707
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:140
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
This is an important base class in LLVM.
Definition: Constant.h:41
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size in bytes, rounded up to a whole number of bytes.
Definition: DataLayout.cpp:728
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition: TypeSize.h:294
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:291
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:302
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:174
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2307
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:257
@ MAX_INT_BITS
Maximum number of bits that can be specified.
Definition: DerivedTypes.h:52
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:177
Value * getPointerOperand()
Definition: Instructions.h:264
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:214
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:220
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
@ LAST_INTEGER_VALUETYPE
@ LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE
@ LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static auto all_valuetypes()
SimpleValueType Iteration.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
MVT getPow2VectorType() const
Widens the length of the given vector MVT up to the nearest power of 2 and returns that type.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool isStatepointSpillSlotObjectIndex(int ObjectIdx) const
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:519
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
A description of a memory reference used in the backend.
unsigned getAddrSpace() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
void freezeReservedRegs(const MachineFunction &)
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Class to represent pointers.
Definition: DerivedTypes.h:632
Analysis providing profile information.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
An instruction for storing to memory.
Definition: Instructions.h:301
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:687
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:558
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
static constexpr size_t npos
Definition: StringRef.h:52
bool isValid() const
Returns true if this iterator is still pointing at a valid entry.
Multiway switch.
Provides information about what library functions are available for the current target.
LegalizeTypeAction getTypeAction(MVT VT) const
void setTypeAction(MVT VT, LegalizeTypeAction Action)
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void initActions()
Initialize all of the actions to default values.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const
Returns the target-specific address of the unsafe stack pointer.
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const
Check whether or not MI needs to be moved close to its uses.
virtual unsigned getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const
Return the maximum amount of bytes allowed to be emitted when padding for alignment.
void setMaximumJumpTableSize(unsigned)
Indicate the maximum number of entries in jump tables.
virtual unsigned getMinimumJumpTableEntries() const
Return lower limit for number of blocks in a jump table.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases, uint64_t Range, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) const
Return true if lowering to a jump table is suitable for a set of case clusters which may contain NumC...
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
virtual bool softPromoteHalfType() const
unsigned getMaximumJumpTableSize() const
Return upper limit for number of entries in a jump table.
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
Value * getDefaultSafeStackPointerLocation(IRBuilderBase &IRB, bool UseTLS) const
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual Align getPrefLoopAlignment(MachineLoop *ML=nullptr) const
Return the preferred loop alignment.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
virtual MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const
Returns preferred type for switch condition.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual bool isJumpTableRelative() const
virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const
Return the type to use for a scalar shift opcode, given the shifted amount type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
unsigned getMinimumJumpTableDensity(bool OptForSize) const
Return lower limit of the density in a jump table.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
TargetLoweringBase(const TargetMachine &TM)
NOTE: The TargetMachine owns TLOF.
LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const
Return pair that represents the legalization kind (first) that needs to happen to EVT (second) in ord...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should should continue looking for chain dependencies when trying to find...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
virtual Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
virtual Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
Inserts in the IR a target-specific intrinsic specifying a fence.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
std::pair< LegalizeTypeAction, EVT > LegalizeKind
LegalizeKind holds the legalization kind that needs to happen to EVT in order to type-legalize it.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
bool isPositionIndependent() const
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
Reloc::Model getRelocationModel() const
Returns the code generation relocation model.
TargetOptions Options
unsigned LoopAlignment
If greater than 0, override TargetLoweringBase::PrefLoopAlignment.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
unsigned getGISelRematGlobalCost() const
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isWindowsGNUEnvironment() const
Definition: Triple.h:617
bool isAndroid() const
Tests whether the target is Android.
Definition: Triple.h:725
@ aarch64_32
Definition: Triple.h:53
bool isOSFreeBSD() const
Definition: Triple.h:545
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
constexpr LeafTy coefficientNextPowerOf2() const
Definition: TypeSize.h:242
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:166
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:163
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition: TypeSize.h:234
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
Definition: CallingConv.h:111
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:749
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:722
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1276
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1303
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:496
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1206
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1208
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1209
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:269
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:486
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:963
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:978
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:779
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1191
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:786
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:542
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1292
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:687
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:816
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1302
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1204
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:898
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1205
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1324
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:773
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:661
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1289
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:721
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1158
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1293
@ ADDCARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:303
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:757
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:923
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1207
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:506
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:726
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1304
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:626
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1297
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:704
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1203
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:599
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1202
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:962
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:776
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1148
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:852
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1185
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1210
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:955
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1152
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:805
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:794
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:673
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:883
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1305
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1200
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:968
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1201
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:832
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1125
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:679
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1145
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:656
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1290
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:279
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:523
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:611
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1199
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:865
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:827
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:902
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:851
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:782
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1277
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:492
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1257
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:313
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1447
static const int LAST_INDEXED_MODE
Definition: ISDOpcodes.h:1398
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:395
EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx)
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition: MathExtras.h:522
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
Definition: Sequence.h:327
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
Definition: Loads.cpp:201
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
constexpr force_iteration_on_noniterable_enum_t force_iteration_on_noniterable_enum
Definition: Sequence.h:108
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:306
bool isReleaseOrStronger(AtomicOrdering AO)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:292
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1833
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ FMul
Product of floats.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
@ FAdd
Sum of floats.
bool isAcquireOrStronger(AtomicOrdering AO)
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:121
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
EVT getPow2VectorType(LLVMContext &Context) const
Widens the length of the given vector EVT up to the nearest power of 2 and returns that type.
Definition: ValueTypes.h:455
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:283
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:333
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition: ValueTypes.h:448
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isFixedLengthVector() const
Definition: ValueTypes.h:170
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
Definition: ValueTypes.h:397
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:306
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:194
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:319
bool isZeroSized() const
Test if the given EVT has zero size, this will fail if called on a scalable type.
Definition: ValueTypes.h:124
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:431
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...