LLVM 19.0.0git
TargetLoweringBase.cpp
Go to the documentation of this file.
1//===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLoweringBase class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/BitVector.h"
14#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/Loads.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/CallingConv.h"
40#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/Function.h"
43#include "llvm/IR/GlobalValue.h"
45#include "llvm/IR/IRBuilder.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/Type.h"
57#include <algorithm>
58#include <cassert>
59#include <cstdint>
60#include <cstring>
61#include <iterator>
62#include <string>
63#include <tuple>
64#include <utility>
65
66using namespace llvm;
67
69 "jump-is-expensive", cl::init(false),
70 cl::desc("Do not create extra branches to split comparison logic."),
72
74 ("min-jump-table-entries", cl::init(4), cl::Hidden,
75 cl::desc("Set minimum number of entries to use a jump table."));
76
78 ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden,
79 cl::desc("Set maximum size of jump tables."));
80
81/// Minimum jump table density for normal functions.
83 JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
84 cl::desc("Minimum density for building a jump table in "
85 "a normal function"));
86
87/// Minimum jump table density for -Os or -Oz functions.
89 "optsize-jump-table-density", cl::init(40), cl::Hidden,
90 cl::desc("Minimum density for building a jump table in "
91 "an optsize function"));
92
93// FIXME: This option is only to test if the strict fp operation processed
94// correctly by preventing mutating strict fp operation to normal fp operation
95// during development. When the backend supports strict float operation, this
96// option will be meaningless.
97static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation",
98 cl::desc("Don't mutate strict-float node to a legalize node"),
99 cl::init(false), cl::Hidden);
100
101static bool darwinHasSinCos(const Triple &TT) {
102 assert(TT.isOSDarwin() && "should be called with darwin triple");
103 // Don't bother with 32 bit x86.
104 if (TT.getArch() == Triple::x86)
105 return false;
106 // Macos < 10.9 has no sincos_stret.
107 if (TT.isMacOSX())
108 return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit();
109 // iOS < 7.0 has no sincos_stret.
110 if (TT.isiOS())
111 return !TT.isOSVersionLT(7, 0);
112 // Any other darwin such as WatchOS/TvOS is new enough.
113 return true;
114}
115
116void TargetLoweringBase::InitLibcalls(const Triple &TT) {
117#define HANDLE_LIBCALL(code, name) \
118 setLibcallName(RTLIB::code, name);
119#include "llvm/IR/RuntimeLibcalls.def"
120#undef HANDLE_LIBCALL
121 // Initialize calling conventions to their default.
122 for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
124
125 // Use the f128 variants of math functions on x86_64
126 if (TT.getArch() == Triple::ArchType::x86_64 && TT.isGNUEnvironment()) {
127 setLibcallName(RTLIB::REM_F128, "fmodf128");
128 setLibcallName(RTLIB::FMA_F128, "fmaf128");
129 setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
130 setLibcallName(RTLIB::CBRT_F128, "cbrtf128");
131 setLibcallName(RTLIB::LOG_F128, "logf128");
132 setLibcallName(RTLIB::LOG_FINITE_F128, "__logf128_finite");
133 setLibcallName(RTLIB::LOG2_F128, "log2f128");
134 setLibcallName(RTLIB::LOG2_FINITE_F128, "__log2f128_finite");
135 setLibcallName(RTLIB::LOG10_F128, "log10f128");
136 setLibcallName(RTLIB::LOG10_FINITE_F128, "__log10f128_finite");
137 setLibcallName(RTLIB::EXP_F128, "expf128");
138 setLibcallName(RTLIB::EXP_FINITE_F128, "__expf128_finite");
139 setLibcallName(RTLIB::EXP2_F128, "exp2f128");
140 setLibcallName(RTLIB::EXP2_FINITE_F128, "__exp2f128_finite");
141 setLibcallName(RTLIB::EXP10_F128, "exp10f128");
142 setLibcallName(RTLIB::SIN_F128, "sinf128");
143 setLibcallName(RTLIB::COS_F128, "cosf128");
144 setLibcallName(RTLIB::TAN_F128, "tanf128");
145 setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
146 setLibcallName(RTLIB::ASIN_F128, "asinf128");
147 setLibcallName(RTLIB::ACOS_F128, "acosf128");
148 setLibcallName(RTLIB::ATAN_F128, "atanf128");
149 setLibcallName(RTLIB::SINH_F128, "sinhf128");
150 setLibcallName(RTLIB::COSH_F128, "coshf128");
151 setLibcallName(RTLIB::TANH_F128, "tanhf128");
152 setLibcallName(RTLIB::POW_F128, "powf128");
153 setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
154 setLibcallName(RTLIB::CEIL_F128, "ceilf128");
155 setLibcallName(RTLIB::TRUNC_F128, "truncf128");
156 setLibcallName(RTLIB::RINT_F128, "rintf128");
157 setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
158 setLibcallName(RTLIB::ROUND_F128, "roundf128");
159 setLibcallName(RTLIB::ROUNDEVEN_F128, "roundevenf128");
160 setLibcallName(RTLIB::FLOOR_F128, "floorf128");
161 setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
162 setLibcallName(RTLIB::FMIN_F128, "fminf128");
163 setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
164 setLibcallName(RTLIB::LROUND_F128, "lroundf128");
165 setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
166 setLibcallName(RTLIB::LRINT_F128, "lrintf128");
167 setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
168 setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
169 setLibcallName(RTLIB::FREXP_F128, "frexpf128");
170 }
171
172 // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
173 if (TT.isPPC()) {
174 setLibcallName(RTLIB::ADD_F128, "__addkf3");
175 setLibcallName(RTLIB::SUB_F128, "__subkf3");
176 setLibcallName(RTLIB::MUL_F128, "__mulkf3");
177 setLibcallName(RTLIB::DIV_F128, "__divkf3");
178 setLibcallName(RTLIB::POWI_F128, "__powikf2");
179 setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
180 setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
181 setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
182 setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
183 setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
184 setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
185 setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
186 setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
187 setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
188 setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
189 setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
190 setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
191 setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
192 setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
193 setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
194 setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
195 setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
196 setLibcallName(RTLIB::UNE_F128, "__nekf2");
197 setLibcallName(RTLIB::OGE_F128, "__gekf2");
198 setLibcallName(RTLIB::OLT_F128, "__ltkf2");
199 setLibcallName(RTLIB::OLE_F128, "__lekf2");
200 setLibcallName(RTLIB::OGT_F128, "__gtkf2");
201 setLibcallName(RTLIB::UO_F128, "__unordkf2");
202 }
203
204 // A few names are different on particular architectures or environments.
205 if (TT.isOSDarwin()) {
206 // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
207 // of the gnueabi-style __gnu_*_ieee.
208 // FIXME: What about other targets?
209 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
210 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
211
212 // Some darwins have an optimized __bzero/bzero function.
213 switch (TT.getArch()) {
214 case Triple::x86:
215 case Triple::x86_64:
216 if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
217 setLibcallName(RTLIB::BZERO, "__bzero");
218 break;
219 case Triple::aarch64:
221 setLibcallName(RTLIB::BZERO, "bzero");
222 break;
223 default:
224 break;
225 }
226
227 if (darwinHasSinCos(TT)) {
228 setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
229 setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
230 if (TT.isWatchABI()) {
231 setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
233 setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
235 }
236 }
237
238 switch (TT.getOS()) {
239 case Triple::MacOSX:
240 if (TT.isMacOSXVersionLT(10, 9)) {
241 setLibcallName(RTLIB::EXP10_F32, nullptr);
242 setLibcallName(RTLIB::EXP10_F64, nullptr);
243 } else {
244 setLibcallName(RTLIB::EXP10_F32, "__exp10f");
245 setLibcallName(RTLIB::EXP10_F64, "__exp10");
246 }
247 break;
248 case Triple::IOS:
249 if (TT.isOSVersionLT(7, 0)) {
250 setLibcallName(RTLIB::EXP10_F32, nullptr);
251 setLibcallName(RTLIB::EXP10_F64, nullptr);
252 break;
253 }
254 [[fallthrough]];
255 case Triple::TvOS:
256 case Triple::WatchOS:
257 case Triple::XROS:
258 setLibcallName(RTLIB::EXP10_F32, "__exp10f");
259 setLibcallName(RTLIB::EXP10_F64, "__exp10");
260 break;
261 default:
262 break;
263 }
264 } else {
265 setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
266 setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
267 }
268
269 if (TT.isGNUEnvironment() || TT.isOSFuchsia() ||
270 (TT.isAndroid() && !TT.isAndroidVersionLT(9))) {
271 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
272 setLibcallName(RTLIB::SINCOS_F64, "sincos");
273 setLibcallName(RTLIB::SINCOS_F80, "sincosl");
274 setLibcallName(RTLIB::SINCOS_F128, "sincosl");
275 setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
276 }
277
278 if (TT.isPS()) {
279 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
280 setLibcallName(RTLIB::SINCOS_F64, "sincos");
281 }
282
283 if (TT.isOSOpenBSD()) {
284 setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
285 }
286
287 if (TT.isOSWindows() && !TT.isOSCygMing()) {
288 setLibcallName(RTLIB::LDEXP_F32, nullptr);
289 setLibcallName(RTLIB::LDEXP_F80, nullptr);
290 setLibcallName(RTLIB::LDEXP_F128, nullptr);
291 setLibcallName(RTLIB::LDEXP_PPCF128, nullptr);
292
293 setLibcallName(RTLIB::FREXP_F32, nullptr);
294 setLibcallName(RTLIB::FREXP_F80, nullptr);
295 setLibcallName(RTLIB::FREXP_F128, nullptr);
296 setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
297 }
298
299 if (TT.isAArch64()) {
300 if (TT.isOSMSVCRT()) {
301 // MSVCRT doesn't have powi; fall back to pow
302 setLibcallName(RTLIB::POWI_F32, nullptr);
303 setLibcallName(RTLIB::POWI_F64, nullptr);
304 }
305 }
306
307 // Disable most libcalls on AMDGPU.
308 if (TT.isAMDGPU()) {
309 for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) {
310 if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
311 setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
312 }
313 }
314
315 // Disable most libcalls on NVPTX.
316 if (TT.isNVPTX()) {
317 for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
318 if (I < RTLIB::ATOMIC_LOAD || I > RTLIB::ATOMIC_FETCH_NAND_16)
319 setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
320 }
321
322 if (TT.isARM() || TT.isThumb()) {
323 // These libcalls are not available in 32-bit.
324 setLibcallName(RTLIB::SHL_I128, nullptr);
325 setLibcallName(RTLIB::SRL_I128, nullptr);
326 setLibcallName(RTLIB::SRA_I128, nullptr);
327 setLibcallName(RTLIB::MUL_I128, nullptr);
328 setLibcallName(RTLIB::MULO_I64, nullptr);
329 setLibcallName(RTLIB::MULO_I128, nullptr);
330
331 if (TT.isOSMSVCRT()) {
332 // MSVCRT doesn't have powi; fall back to pow
333 setLibcallName(RTLIB::POWI_F32, nullptr);
334 setLibcallName(RTLIB::POWI_F64, nullptr);
335 }
336 }
337
338 if (TT.getArch() == Triple::ArchType::avr) {
339 // Division rtlib functions (not supported), use divmod functions instead
340 setLibcallName(RTLIB::SDIV_I8, nullptr);
341 setLibcallName(RTLIB::SDIV_I16, nullptr);
342 setLibcallName(RTLIB::SDIV_I32, nullptr);
343 setLibcallName(RTLIB::UDIV_I8, nullptr);
344 setLibcallName(RTLIB::UDIV_I16, nullptr);
345 setLibcallName(RTLIB::UDIV_I32, nullptr);
346
347 // Modulus rtlib functions (not supported), use divmod functions instead
348 setLibcallName(RTLIB::SREM_I8, nullptr);
349 setLibcallName(RTLIB::SREM_I16, nullptr);
350 setLibcallName(RTLIB::SREM_I32, nullptr);
351 setLibcallName(RTLIB::UREM_I8, nullptr);
352 setLibcallName(RTLIB::UREM_I16, nullptr);
353 setLibcallName(RTLIB::UREM_I32, nullptr);
354 }
355
356 if (TT.getArch() == Triple::ArchType::hexagon) {
357 // These cause problems when the shift amount is non-constant.
358 setLibcallName(RTLIB::SHL_I128, nullptr);
359 setLibcallName(RTLIB::SRL_I128, nullptr);
360 setLibcallName(RTLIB::SRA_I128, nullptr);
361 }
362
363 if (TT.isLoongArch()) {
364 if (!TT.isLoongArch64()) {
365 // Set libcalls.
366 setLibcallName(RTLIB::MUL_I128, nullptr);
367 // The MULO libcall is not part of libgcc, only compiler-rt.
368 setLibcallName(RTLIB::MULO_I64, nullptr);
369 }
370 // The MULO libcall is not part of libgcc, only compiler-rt.
371 setLibcallName(RTLIB::MULO_I128, nullptr);
372 }
373
374 if (TT.isMIPS32()) {
375 // These libcalls are not available in 32-bit.
376 setLibcallName(RTLIB::SHL_I128, nullptr);
377 setLibcallName(RTLIB::SRL_I128, nullptr);
378 setLibcallName(RTLIB::SRA_I128, nullptr);
379 setLibcallName(RTLIB::MUL_I128, nullptr);
380 setLibcallName(RTLIB::MULO_I64, nullptr);
381 setLibcallName(RTLIB::MULO_I128, nullptr);
382 }
383
384 if (TT.isPPC()) {
385 if (!TT.isPPC64()) {
386 // These libcalls are not available in 32-bit.
387 setLibcallName(RTLIB::SHL_I128, nullptr);
388 setLibcallName(RTLIB::SRL_I128, nullptr);
389 setLibcallName(RTLIB::SRA_I128, nullptr);
390 setLibcallName(RTLIB::MUL_I128, nullptr);
391 setLibcallName(RTLIB::MULO_I64, nullptr);
392 }
393 setLibcallName(RTLIB::MULO_I128, nullptr);
394 }
395
396 if (TT.isRISCV32()) {
397 // These libcalls are not available in 32-bit.
398 setLibcallName(RTLIB::SHL_I128, nullptr);
399 setLibcallName(RTLIB::SRL_I128, nullptr);
400 setLibcallName(RTLIB::SRA_I128, nullptr);
401 setLibcallName(RTLIB::MUL_I128, nullptr);
402 setLibcallName(RTLIB::MULO_I64, nullptr);
403 }
404
405 if (TT.isSPARC()) {
406 if (!TT.isSPARC64()) {
407 // These libcalls are not available in 32-bit.
408 setLibcallName(RTLIB::MULO_I64, nullptr);
409 setLibcallName(RTLIB::MUL_I128, nullptr);
410 setLibcallName(RTLIB::SHL_I128, nullptr);
411 setLibcallName(RTLIB::SRL_I128, nullptr);
412 setLibcallName(RTLIB::SRA_I128, nullptr);
413 }
414 setLibcallName(RTLIB::MULO_I128, nullptr);
415 }
416
417 if (TT.isSystemZ()) {
418 setLibcallName(RTLIB::SRL_I128, nullptr);
419 setLibcallName(RTLIB::SHL_I128, nullptr);
420 setLibcallName(RTLIB::SRA_I128, nullptr);
421 }
422
423 if (TT.isX86()) {
424 if (TT.getArch() == Triple::ArchType::x86) {
425 // These libcalls are not available in 32-bit.
426 setLibcallName(RTLIB::SHL_I128, nullptr);
427 setLibcallName(RTLIB::SRL_I128, nullptr);
428 setLibcallName(RTLIB::SRA_I128, nullptr);
429 setLibcallName(RTLIB::MUL_I128, nullptr);
430 // The MULO libcall is not part of libgcc, only compiler-rt.
431 setLibcallName(RTLIB::MULO_I64, nullptr);
432 }
433
434 // The MULO libcall is not part of libgcc, only compiler-rt.
435 setLibcallName(RTLIB::MULO_I128, nullptr);
436
437 if (TT.isOSMSVCRT()) {
438 // MSVCRT doesn't have powi; fall back to pow
439 setLibcallName(RTLIB::POWI_F32, nullptr);
440 setLibcallName(RTLIB::POWI_F64, nullptr);
441 }
442 }
443}
444
445/// GetFPLibCall - Helper to return the right libcall for the given floating
446/// point type, or UNKNOWN_LIBCALL if there is none.
448 RTLIB::Libcall Call_F32,
449 RTLIB::Libcall Call_F64,
450 RTLIB::Libcall Call_F80,
451 RTLIB::Libcall Call_F128,
452 RTLIB::Libcall Call_PPCF128) {
453 return
454 VT == MVT::f32 ? Call_F32 :
455 VT == MVT::f64 ? Call_F64 :
456 VT == MVT::f80 ? Call_F80 :
457 VT == MVT::f128 ? Call_F128 :
458 VT == MVT::ppcf128 ? Call_PPCF128 :
459 RTLIB::UNKNOWN_LIBCALL;
460}
461
462/// getFPEXT - Return the FPEXT_*_* value for the given types, or
463/// UNKNOWN_LIBCALL if there is none.
465 if (OpVT == MVT::f16) {
466 if (RetVT == MVT::f32)
467 return FPEXT_F16_F32;
468 if (RetVT == MVT::f64)
469 return FPEXT_F16_F64;
470 if (RetVT == MVT::f80)
471 return FPEXT_F16_F80;
472 if (RetVT == MVT::f128)
473 return FPEXT_F16_F128;
474 } else if (OpVT == MVT::f32) {
475 if (RetVT == MVT::f64)
476 return FPEXT_F32_F64;
477 if (RetVT == MVT::f128)
478 return FPEXT_F32_F128;
479 if (RetVT == MVT::ppcf128)
480 return FPEXT_F32_PPCF128;
481 } else if (OpVT == MVT::f64) {
482 if (RetVT == MVT::f128)
483 return FPEXT_F64_F128;
484 else if (RetVT == MVT::ppcf128)
485 return FPEXT_F64_PPCF128;
486 } else if (OpVT == MVT::f80) {
487 if (RetVT == MVT::f128)
488 return FPEXT_F80_F128;
489 } else if (OpVT == MVT::bf16) {
490 if (RetVT == MVT::f32)
491 return FPEXT_BF16_F32;
492 }
493
494 return UNKNOWN_LIBCALL;
495}
496
497/// getFPROUND - Return the FPROUND_*_* value for the given types, or
498/// UNKNOWN_LIBCALL if there is none.
500 if (RetVT == MVT::f16) {
501 if (OpVT == MVT::f32)
502 return FPROUND_F32_F16;
503 if (OpVT == MVT::f64)
504 return FPROUND_F64_F16;
505 if (OpVT == MVT::f80)
506 return FPROUND_F80_F16;
507 if (OpVT == MVT::f128)
508 return FPROUND_F128_F16;
509 if (OpVT == MVT::ppcf128)
510 return FPROUND_PPCF128_F16;
511 } else if (RetVT == MVT::bf16) {
512 if (OpVT == MVT::f32)
513 return FPROUND_F32_BF16;
514 if (OpVT == MVT::f64)
515 return FPROUND_F64_BF16;
516 } else if (RetVT == MVT::f32) {
517 if (OpVT == MVT::f64)
518 return FPROUND_F64_F32;
519 if (OpVT == MVT::f80)
520 return FPROUND_F80_F32;
521 if (OpVT == MVT::f128)
522 return FPROUND_F128_F32;
523 if (OpVT == MVT::ppcf128)
524 return FPROUND_PPCF128_F32;
525 } else if (RetVT == MVT::f64) {
526 if (OpVT == MVT::f80)
527 return FPROUND_F80_F64;
528 if (OpVT == MVT::f128)
529 return FPROUND_F128_F64;
530 if (OpVT == MVT::ppcf128)
531 return FPROUND_PPCF128_F64;
532 } else if (RetVT == MVT::f80) {
533 if (OpVT == MVT::f128)
534 return FPROUND_F128_F80;
535 }
536
537 return UNKNOWN_LIBCALL;
538}
539
540/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
541/// UNKNOWN_LIBCALL if there is none.
543 if (OpVT == MVT::f16) {
544 if (RetVT == MVT::i32)
545 return FPTOSINT_F16_I32;
546 if (RetVT == MVT::i64)
547 return FPTOSINT_F16_I64;
548 if (RetVT == MVT::i128)
549 return FPTOSINT_F16_I128;
550 } else if (OpVT == MVT::f32) {
551 if (RetVT == MVT::i32)
552 return FPTOSINT_F32_I32;
553 if (RetVT == MVT::i64)
554 return FPTOSINT_F32_I64;
555 if (RetVT == MVT::i128)
556 return FPTOSINT_F32_I128;
557 } else if (OpVT == MVT::f64) {
558 if (RetVT == MVT::i32)
559 return FPTOSINT_F64_I32;
560 if (RetVT == MVT::i64)
561 return FPTOSINT_F64_I64;
562 if (RetVT == MVT::i128)
563 return FPTOSINT_F64_I128;
564 } else if (OpVT == MVT::f80) {
565 if (RetVT == MVT::i32)
566 return FPTOSINT_F80_I32;
567 if (RetVT == MVT::i64)
568 return FPTOSINT_F80_I64;
569 if (RetVT == MVT::i128)
570 return FPTOSINT_F80_I128;
571 } else if (OpVT == MVT::f128) {
572 if (RetVT == MVT::i32)
573 return FPTOSINT_F128_I32;
574 if (RetVT == MVT::i64)
575 return FPTOSINT_F128_I64;
576 if (RetVT == MVT::i128)
577 return FPTOSINT_F128_I128;
578 } else if (OpVT == MVT::ppcf128) {
579 if (RetVT == MVT::i32)
580 return FPTOSINT_PPCF128_I32;
581 if (RetVT == MVT::i64)
582 return FPTOSINT_PPCF128_I64;
583 if (RetVT == MVT::i128)
584 return FPTOSINT_PPCF128_I128;
585 }
586 return UNKNOWN_LIBCALL;
587}
588
589/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
590/// UNKNOWN_LIBCALL if there is none.
592 if (OpVT == MVT::f16) {
593 if (RetVT == MVT::i32)
594 return FPTOUINT_F16_I32;
595 if (RetVT == MVT::i64)
596 return FPTOUINT_F16_I64;
597 if (RetVT == MVT::i128)
598 return FPTOUINT_F16_I128;
599 } else if (OpVT == MVT::f32) {
600 if (RetVT == MVT::i32)
601 return FPTOUINT_F32_I32;
602 if (RetVT == MVT::i64)
603 return FPTOUINT_F32_I64;
604 if (RetVT == MVT::i128)
605 return FPTOUINT_F32_I128;
606 } else if (OpVT == MVT::f64) {
607 if (RetVT == MVT::i32)
608 return FPTOUINT_F64_I32;
609 if (RetVT == MVT::i64)
610 return FPTOUINT_F64_I64;
611 if (RetVT == MVT::i128)
612 return FPTOUINT_F64_I128;
613 } else if (OpVT == MVT::f80) {
614 if (RetVT == MVT::i32)
615 return FPTOUINT_F80_I32;
616 if (RetVT == MVT::i64)
617 return FPTOUINT_F80_I64;
618 if (RetVT == MVT::i128)
619 return FPTOUINT_F80_I128;
620 } else if (OpVT == MVT::f128) {
621 if (RetVT == MVT::i32)
622 return FPTOUINT_F128_I32;
623 if (RetVT == MVT::i64)
624 return FPTOUINT_F128_I64;
625 if (RetVT == MVT::i128)
626 return FPTOUINT_F128_I128;
627 } else if (OpVT == MVT::ppcf128) {
628 if (RetVT == MVT::i32)
629 return FPTOUINT_PPCF128_I32;
630 if (RetVT == MVT::i64)
631 return FPTOUINT_PPCF128_I64;
632 if (RetVT == MVT::i128)
633 return FPTOUINT_PPCF128_I128;
634 }
635 return UNKNOWN_LIBCALL;
636}
637
638/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
639/// UNKNOWN_LIBCALL if there is none.
641 if (OpVT == MVT::i32) {
642 if (RetVT == MVT::f16)
643 return SINTTOFP_I32_F16;
644 if (RetVT == MVT::f32)
645 return SINTTOFP_I32_F32;
646 if (RetVT == MVT::f64)
647 return SINTTOFP_I32_F64;
648 if (RetVT == MVT::f80)
649 return SINTTOFP_I32_F80;
650 if (RetVT == MVT::f128)
651 return SINTTOFP_I32_F128;
652 if (RetVT == MVT::ppcf128)
653 return SINTTOFP_I32_PPCF128;
654 } else if (OpVT == MVT::i64) {
655 if (RetVT == MVT::f16)
656 return SINTTOFP_I64_F16;
657 if (RetVT == MVT::f32)
658 return SINTTOFP_I64_F32;
659 if (RetVT == MVT::f64)
660 return SINTTOFP_I64_F64;
661 if (RetVT == MVT::f80)
662 return SINTTOFP_I64_F80;
663 if (RetVT == MVT::f128)
664 return SINTTOFP_I64_F128;
665 if (RetVT == MVT::ppcf128)
666 return SINTTOFP_I64_PPCF128;
667 } else if (OpVT == MVT::i128) {
668 if (RetVT == MVT::f16)
669 return SINTTOFP_I128_F16;
670 if (RetVT == MVT::f32)
671 return SINTTOFP_I128_F32;
672 if (RetVT == MVT::f64)
673 return SINTTOFP_I128_F64;
674 if (RetVT == MVT::f80)
675 return SINTTOFP_I128_F80;
676 if (RetVT == MVT::f128)
677 return SINTTOFP_I128_F128;
678 if (RetVT == MVT::ppcf128)
679 return SINTTOFP_I128_PPCF128;
680 }
681 return UNKNOWN_LIBCALL;
682}
683
684/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
685/// UNKNOWN_LIBCALL if there is none.
687 if (OpVT == MVT::i32) {
688 if (RetVT == MVT::f16)
689 return UINTTOFP_I32_F16;
690 if (RetVT == MVT::f32)
691 return UINTTOFP_I32_F32;
692 if (RetVT == MVT::f64)
693 return UINTTOFP_I32_F64;
694 if (RetVT == MVT::f80)
695 return UINTTOFP_I32_F80;
696 if (RetVT == MVT::f128)
697 return UINTTOFP_I32_F128;
698 if (RetVT == MVT::ppcf128)
699 return UINTTOFP_I32_PPCF128;
700 } else if (OpVT == MVT::i64) {
701 if (RetVT == MVT::f16)
702 return UINTTOFP_I64_F16;
703 if (RetVT == MVT::f32)
704 return UINTTOFP_I64_F32;
705 if (RetVT == MVT::f64)
706 return UINTTOFP_I64_F64;
707 if (RetVT == MVT::f80)
708 return UINTTOFP_I64_F80;
709 if (RetVT == MVT::f128)
710 return UINTTOFP_I64_F128;
711 if (RetVT == MVT::ppcf128)
712 return UINTTOFP_I64_PPCF128;
713 } else if (OpVT == MVT::i128) {
714 if (RetVT == MVT::f16)
715 return UINTTOFP_I128_F16;
716 if (RetVT == MVT::f32)
717 return UINTTOFP_I128_F32;
718 if (RetVT == MVT::f64)
719 return UINTTOFP_I128_F64;
720 if (RetVT == MVT::f80)
721 return UINTTOFP_I128_F80;
722 if (RetVT == MVT::f128)
723 return UINTTOFP_I128_F128;
724 if (RetVT == MVT::ppcf128)
725 return UINTTOFP_I128_PPCF128;
726 }
727 return UNKNOWN_LIBCALL;
728}
729
731 return getFPLibCall(RetVT, POWI_F32, POWI_F64, POWI_F80, POWI_F128,
732 POWI_PPCF128);
733}
734
736 return getFPLibCall(RetVT, LDEXP_F32, LDEXP_F64, LDEXP_F80, LDEXP_F128,
737 LDEXP_PPCF128);
738}
739
741 return getFPLibCall(RetVT, FREXP_F32, FREXP_F64, FREXP_F80, FREXP_F128,
742 FREXP_PPCF128);
743}
744
746 AtomicOrdering Order,
747 uint64_t MemSize) {
748 unsigned ModeN, ModelN;
749 switch (MemSize) {
750 case 1:
751 ModeN = 0;
752 break;
753 case 2:
754 ModeN = 1;
755 break;
756 case 4:
757 ModeN = 2;
758 break;
759 case 8:
760 ModeN = 3;
761 break;
762 case 16:
763 ModeN = 4;
764 break;
765 default:
766 return RTLIB::UNKNOWN_LIBCALL;
767 }
768
769 switch (Order) {
771 ModelN = 0;
772 break;
774 ModelN = 1;
775 break;
777 ModelN = 2;
778 break;
781 ModelN = 3;
782 break;
783 default:
784 return UNKNOWN_LIBCALL;
785 }
786
787 return LC[ModeN][ModelN];
788}
789
791 MVT VT) {
792 if (!VT.isScalarInteger())
793 return UNKNOWN_LIBCALL;
794 uint64_t MemSize = VT.getScalarSizeInBits() / 8;
795
796#define LCALLS(A, B) \
797 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
798#define LCALL5(A) \
799 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
800 switch (Opc) {
802 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};
803 return getOutlineAtomicHelper(LC, Order, MemSize);
804 }
805 case ISD::ATOMIC_SWAP: {
806 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};
807 return getOutlineAtomicHelper(LC, Order, MemSize);
808 }
810 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};
811 return getOutlineAtomicHelper(LC, Order, MemSize);
812 }
813 case ISD::ATOMIC_LOAD_OR: {
814 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};
815 return getOutlineAtomicHelper(LC, Order, MemSize);
816 }
818 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};
819 return getOutlineAtomicHelper(LC, Order, MemSize);
820 }
822 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};
823 return getOutlineAtomicHelper(LC, Order, MemSize);
824 }
825 default:
826 return UNKNOWN_LIBCALL;
827 }
828#undef LCALLS
829#undef LCALL5
830}
831
833#define OP_TO_LIBCALL(Name, Enum) \
834 case Name: \
835 switch (VT.SimpleTy) { \
836 default: \
837 return UNKNOWN_LIBCALL; \
838 case MVT::i8: \
839 return Enum##_1; \
840 case MVT::i16: \
841 return Enum##_2; \
842 case MVT::i32: \
843 return Enum##_4; \
844 case MVT::i64: \
845 return Enum##_8; \
846 case MVT::i128: \
847 return Enum##_16; \
848 }
849
850 switch (Opc) {
851 OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET)
852 OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP)
853 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD)
854 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB)
855 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND)
856 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR)
857 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR)
858 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND)
859 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX)
860 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX)
861 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN)
862 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN)
863 }
864
865#undef OP_TO_LIBCALL
866
867 return UNKNOWN_LIBCALL;
868}
869
871 switch (ElementSize) {
872 case 1:
873 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1;
874 case 2:
875 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2;
876 case 4:
877 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4;
878 case 8:
879 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8;
880 case 16:
881 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16;
882 default:
883 return UNKNOWN_LIBCALL;
884 }
885}
886
888 switch (ElementSize) {
889 case 1:
890 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;
891 case 2:
892 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;
893 case 4:
894 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;
895 case 8:
896 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;
897 case 16:
898 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;
899 default:
900 return UNKNOWN_LIBCALL;
901 }
902}
903
905 switch (ElementSize) {
906 case 1:
907 return MEMSET_ELEMENT_UNORDERED_ATOMIC_1;
908 case 2:
909 return MEMSET_ELEMENT_UNORDERED_ATOMIC_2;
910 case 4:
911 return MEMSET_ELEMENT_UNORDERED_ATOMIC_4;
912 case 8:
913 return MEMSET_ELEMENT_UNORDERED_ATOMIC_8;
914 case 16:
915 return MEMSET_ELEMENT_UNORDERED_ATOMIC_16;
916 default:
917 return UNKNOWN_LIBCALL;
918 }
919}
920
921/// InitCmpLibcallCCs - Set default comparison libcall CC.
923 std::fill(CCs, CCs + RTLIB::UNKNOWN_LIBCALL, ISD::SETCC_INVALID);
924 CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
925 CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
926 CCs[RTLIB::OEQ_F128] = ISD::SETEQ;
927 CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
928 CCs[RTLIB::UNE_F32] = ISD::SETNE;
929 CCs[RTLIB::UNE_F64] = ISD::SETNE;
930 CCs[RTLIB::UNE_F128] = ISD::SETNE;
931 CCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
932 CCs[RTLIB::OGE_F32] = ISD::SETGE;
933 CCs[RTLIB::OGE_F64] = ISD::SETGE;
934 CCs[RTLIB::OGE_F128] = ISD::SETGE;
935 CCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
936 CCs[RTLIB::OLT_F32] = ISD::SETLT;
937 CCs[RTLIB::OLT_F64] = ISD::SETLT;
938 CCs[RTLIB::OLT_F128] = ISD::SETLT;
939 CCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
940 CCs[RTLIB::OLE_F32] = ISD::SETLE;
941 CCs[RTLIB::OLE_F64] = ISD::SETLE;
942 CCs[RTLIB::OLE_F128] = ISD::SETLE;
943 CCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
944 CCs[RTLIB::OGT_F32] = ISD::SETGT;
945 CCs[RTLIB::OGT_F64] = ISD::SETGT;
946 CCs[RTLIB::OGT_F128] = ISD::SETGT;
947 CCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
948 CCs[RTLIB::UO_F32] = ISD::SETNE;
949 CCs[RTLIB::UO_F64] = ISD::SETNE;
950 CCs[RTLIB::UO_F128] = ISD::SETNE;
951 CCs[RTLIB::UO_PPCF128] = ISD::SETNE;
952}
953
954/// NOTE: The TargetMachine owns TLOF.
956 initActions();
957
958 // Perform these initializations only once.
964 HasMultipleConditionRegisters = false;
965 HasExtractBitsInsn = false;
966 JumpIsExpensive = JumpIsExpensiveOverride;
968 EnableExtLdPromotion = false;
969 StackPointerRegisterToSaveRestore = 0;
970 BooleanContents = UndefinedBooleanContent;
971 BooleanFloatContents = UndefinedBooleanContent;
972 BooleanVectorContents = UndefinedBooleanContent;
973 SchedPreferenceInfo = Sched::ILP;
976 MaxBytesForAlignment = 0;
977 MaxAtomicSizeInBitsSupported = 0;
978
979 // Assume that even with libcalls, no target supports wider than 128 bit
980 // division.
981 MaxDivRemBitWidthSupported = 128;
982
983 MaxLargeFPConvertBitWidthSupported = llvm::IntegerType::MAX_INT_BITS;
984
985 MinCmpXchgSizeInBits = 0;
986 SupportsUnalignedAtomics = false;
987
988 std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
989
990 InitLibcalls(TM.getTargetTriple());
991 InitCmpLibcallCCs(CmpLibcallCCs);
992}
993
995 // All operations default to being supported.
996 memset(OpActions, 0, sizeof(OpActions));
997 memset(LoadExtActions, 0, sizeof(LoadExtActions));
998 memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
999 memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
1000 memset(CondCodeActions, 0, sizeof(CondCodeActions));
1001 std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
1002 std::fill(std::begin(TargetDAGCombineArray),
1003 std::end(TargetDAGCombineArray), 0);
1004
1005 // Let extending atomic loads be unsupported by default.
1006 for (MVT ValVT : MVT::all_valuetypes())
1007 for (MVT MemVT : MVT::all_valuetypes())
1009 Expand);
1010
1011 // We're somewhat special casing MVT::i2 and MVT::i4. Ideally we want to
1012 // remove this and targets should individually set these types if not legal.
1015 for (MVT VT : {MVT::i2, MVT::i4})
1016 OpActions[(unsigned)VT.SimpleTy][NT] = Expand;
1017 }
1018 for (MVT AVT : MVT::all_valuetypes()) {
1019 for (MVT VT : {MVT::i2, MVT::i4, MVT::v128i2, MVT::v64i4}) {
1020 setTruncStoreAction(AVT, VT, Expand);
1023 }
1024 }
1025 for (unsigned IM = (unsigned)ISD::PRE_INC;
1026 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
1027 for (MVT VT : {MVT::i2, MVT::i4}) {
1032 }
1033 }
1034
1035 for (MVT VT : MVT::fp_valuetypes()) {
1036 MVT IntVT = MVT::getIntegerVT(VT.getFixedSizeInBits());
1037 if (IntVT.isValid()) {
1040 }
1041 }
1042
1043 // Set default actions for various operations.
1044 for (MVT VT : MVT::all_valuetypes()) {
1045 // Default all indexed load / store to expand.
1046 for (unsigned IM = (unsigned)ISD::PRE_INC;
1047 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
1052 }
1053
1054 // Most backends expect to see the node which just returns the value loaded.
1056
1057 // These operations default to expand.
1075 VT, Expand);
1076
1077 // Overflow operations default to expand
1080 VT, Expand);
1081
1082 // Carry-using overflow operations default to expand.
1085 VT, Expand);
1086
1087 // ADDC/ADDE/SUBC/SUBE default to expand.
1089 Expand);
1090
1091 // [US]CMP default to expand
1093
1094 // Halving adds
1097 Expand);
1098
1099 // Absolute difference
1101
1102 // These default to Expand so they will be expanded to CTLZ/CTTZ by default.
1104 Expand);
1105
1107
1108 // These library functions default to expand.
1110 Expand);
1111
1112 // These operations default to expand for vector types.
1113 if (VT.isVector())
1119 VT, Expand);
1120
1121 // Constrained floating-point operations default to expand.
1122#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1123 setOperationAction(ISD::STRICT_##DAGN, VT, Expand);
1124#include "llvm/IR/ConstrainedOps.def"
1125
1126 // For most targets @llvm.get.dynamic.area.offset just returns 0.
1128
1129 // Vector reduction default to expand.
1137 VT, Expand);
1138
1139 // Named vector shuffles default to expand.
1141
1142 // VP operations default to expand.
1143#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
1144 setOperationAction(ISD::SDOPC, VT, Expand);
1145#include "llvm/IR/VPIntrinsics.def"
1146
1147 // FP environment operations default to expand.
1151 }
1152
1153 // Most targets ignore the @llvm.prefetch intrinsic.
1155
1156 // Most targets also ignore the @llvm.readcyclecounter intrinsic.
1158
1159 // Most targets also ignore the @llvm.readsteadycounter intrinsic.
1161
1162 // ConstantFP nodes default to expand. Targets can either change this to
1163 // Legal, in which case all fp constants are legal, or use isFPImmLegal()
1164 // to optimize expansions for certain constants.
1166 {MVT::bf16, MVT::f16, MVT::f32, MVT::f64, MVT::f80, MVT::f128},
1167 Expand);
1168
1169 // These library functions default to expand.
1176 {MVT::f32, MVT::f64, MVT::f128}, Expand);
1177
1180 MVT::f16, Promote);
1181 // Default ISD::TRAP to expand (which turns it into abort).
1182 setOperationAction(ISD::TRAP, MVT::Other, Expand);
1183
1184 // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
1185 // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
1187
1189
1192
1193 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
1196 }
1198
1199 // This one by default will call __clear_cache unless the target
1200 // wants something different.
1202}
1203
1205 EVT) const {
1206 return MVT::getIntegerVT(DL.getPointerSizeInBits(0));
1207}
1208
1210 const DataLayout &DL) const {
1211 assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
1212 if (LHSTy.isVector())
1213 return LHSTy;
1214 MVT ShiftVT = getScalarShiftAmountTy(DL, LHSTy);
1215 // If any possible shift value won't fit in the prefered type, just use
1216 // something safe. Assume it will be legalized when the shift is expanded.
1217 if (ShiftVT.getSizeInBits() < Log2_32_Ceil(LHSTy.getSizeInBits()))
1218 ShiftVT = MVT::i32;
1219 assert(ShiftVT.getSizeInBits() >= Log2_32_Ceil(LHSTy.getSizeInBits()) &&
1220 "ShiftVT is still too small!");
1221 return ShiftVT;
1222}
1223
1224bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
1225 assert(isTypeLegal(VT));
1226 switch (Op) {
1227 default:
1228 return false;
1229 case ISD::SDIV:
1230 case ISD::UDIV:
1231 case ISD::SREM:
1232 case ISD::UREM:
1233 return true;
1234 }
1235}
1236
1238 unsigned DestAS) const {
1239 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
1240}
1241
1243 Type *RetTy, ElementCount EC, bool ZeroIsPoison,
1244 const ConstantRange *VScaleRange) const {
1245 // Find the smallest "sensible" element type to use for the expansion.
1246 ConstantRange CR(APInt(64, EC.getKnownMinValue()));
1247 if (EC.isScalable())
1248 CR = CR.umul_sat(*VScaleRange);
1249
1250 if (ZeroIsPoison)
1251 CR = CR.subtract(APInt(64, 1));
1252
1253 unsigned EltWidth = RetTy->getScalarSizeInBits();
1254 EltWidth = std::min(EltWidth, (unsigned)CR.getActiveBits());
1255 EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);
1256
1257 return EltWidth;
1258}
1259
1261 // If the command-line option was specified, ignore this request.
1262 if (!JumpIsExpensiveOverride.getNumOccurrences())
1263 JumpIsExpensive = isExpensive;
1264}
1265
1268 // If this is a simple type, use the ComputeRegisterProp mechanism.
1269 if (VT.isSimple()) {
1270 MVT SVT = VT.getSimpleVT();
1271 assert((unsigned)SVT.SimpleTy < std::size(TransformToType));
1272 MVT NVT = TransformToType[SVT.SimpleTy];
1273 LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);
1274
1275 assert((LA == TypeLegal || LA == TypeSoftenFloat ||
1276 LA == TypeSoftPromoteHalf ||
1277 (NVT.isVector() ||
1278 ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger)) &&
1279 "Promote may not follow Expand or Promote");
1280
1281 if (LA == TypeSplitVector)
1282 return LegalizeKind(LA, EVT(SVT).getHalfNumVectorElementsVT(Context));
1283 if (LA == TypeScalarizeVector)
1284 return LegalizeKind(LA, SVT.getVectorElementType());
1285 return LegalizeKind(LA, NVT);
1286 }
1287
1288 // Handle Extended Scalar Types.
1289 if (!VT.isVector()) {
1290 assert(VT.isInteger() && "Float types must be simple");
1291 unsigned BitSize = VT.getSizeInBits();
1292 // First promote to a power-of-two size, then expand if necessary.
1293 if (BitSize < 8 || !isPowerOf2_32(BitSize)) {
1294 EVT NVT = VT.getRoundIntegerType(Context);
1295 assert(NVT != VT && "Unable to round integer VT");
1296 LegalizeKind NextStep = getTypeConversion(Context, NVT);
1297 // Avoid multi-step promotion.
1298 if (NextStep.first == TypePromoteInteger)
1299 return NextStep;
1300 // Return rounded integer type.
1301 return LegalizeKind(TypePromoteInteger, NVT);
1302 }
1303
1305 EVT::getIntegerVT(Context, VT.getSizeInBits() / 2));
1306 }
1307
1308 // Handle vector types.
1309 ElementCount NumElts = VT.getVectorElementCount();
1310 EVT EltVT = VT.getVectorElementType();
1311
1312 // Vectors with only one element are always scalarized.
1313 if (NumElts.isScalar())
1314 return LegalizeKind(TypeScalarizeVector, EltVT);
1315
1316 // Try to widen vector elements until the element type is a power of two and
1317 // promote it to a legal type later on, for example:
1318 // <3 x i8> -> <4 x i8> -> <4 x i32>
1319 if (EltVT.isInteger()) {
1320 // Vectors with a number of elements that is not a power of two are always
1321 // widened, for example <3 x i8> -> <4 x i8>.
1322 if (!VT.isPow2VectorType()) {
1323 NumElts = NumElts.coefficientNextPowerOf2();
1324 EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts);
1325 return LegalizeKind(TypeWidenVector, NVT);
1326 }
1327
1328 // Examine the element type.
1329 LegalizeKind LK = getTypeConversion(Context, EltVT);
1330
1331 // If type is to be expanded, split the vector.
1332 // <4 x i140> -> <2 x i140>
1333 if (LK.first == TypeExpandInteger) {
1337 VT.getHalfNumVectorElementsVT(Context));
1338 }
1339
1340 // Promote the integer element types until a legal vector type is found
1341 // or until the element integer type is too big. If a legal type was not
1342 // found, fallback to the usual mechanism of widening/splitting the
1343 // vector.
1344 EVT OldEltVT = EltVT;
1345 while (true) {
1346 // Increase the bitwidth of the element to the next pow-of-two
1347 // (which is greater than 8 bits).
1348 EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits())
1349 .getRoundIntegerType(Context);
1350
1351 // Stop trying when getting a non-simple element type.
1352 // Note that vector elements may be greater than legal vector element
1353 // types. Example: X86 XMM registers hold 64bit element on 32bit
1354 // systems.
1355 if (!EltVT.isSimple())
1356 break;
1357
1358 // Build a new vector type and check if it is legal.
1359 MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
1360 // Found a legal promoted vector type.
1361 if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal)
1363 EVT::getVectorVT(Context, EltVT, NumElts));
1364 }
1365
1366 // Reset the type to the unexpanded type if we did not find a legal vector
1367 // type with a promoted vector element type.
1368 EltVT = OldEltVT;
1369 }
1370
1371 // Try to widen the vector until a legal type is found.
1372 // If there is no wider legal type, split the vector.
1373 while (true) {
1374 // Round up to the next power of 2.
1375 NumElts = NumElts.coefficientNextPowerOf2();
1376
1377 // If there is no simple vector type with this many elements then there
1378 // cannot be a larger legal vector type. Note that this assumes that
1379 // there are no skipped intermediate vector types in the simple types.
1380 if (!EltVT.isSimple())
1381 break;
1382 MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
1383 if (LargerVector == MVT())
1384 break;
1385
1386 // If this type is legal then widen the vector.
1387 if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal)
1388 return LegalizeKind(TypeWidenVector, LargerVector);
1389 }
1390
1391 // Widen odd vectors to next power of two.
1392 if (!VT.isPow2VectorType()) {
1393 EVT NVT = VT.getPow2VectorType(Context);
1394 return LegalizeKind(TypeWidenVector, NVT);
1395 }
1396
1399
1400 // Vectors with illegal element types are expanded.
1401 EVT NVT = EVT::getVectorVT(Context, EltVT,
1403 return LegalizeKind(TypeSplitVector, NVT);
1404}
1405
1406static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
1407 unsigned &NumIntermediates,
1408 MVT &RegisterVT,
1409 TargetLoweringBase *TLI) {
1410 // Figure out the right, legal destination reg to copy into.
1412 MVT EltTy = VT.getVectorElementType();
1413
1414 unsigned NumVectorRegs = 1;
1415
1416 // Scalable vectors cannot be scalarized, so splitting or widening is
1417 // required.
1418 if (VT.isScalableVector() && !isPowerOf2_32(EC.getKnownMinValue()))
1420 "Splitting or widening of non-power-of-2 MVTs is not implemented.");
1421
1422 // FIXME: We don't support non-power-of-2-sized vectors for now.
1423 // Ideally we could break down into LHS/RHS like LegalizeDAG does.
1424 if (!isPowerOf2_32(EC.getKnownMinValue())) {
1425 // Split EC to unit size (scalable property is preserved).
1426 NumVectorRegs = EC.getKnownMinValue();
1427 EC = ElementCount::getFixed(1);
1428 }
1429
1430 // Divide the input until we get to a supported size. This will
1431 // always end up with an EC that represent a scalar or a scalable
1432 // scalar.
1433 while (EC.getKnownMinValue() > 1 &&
1434 !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) {
1435 EC = EC.divideCoefficientBy(2);
1436 NumVectorRegs <<= 1;
1437 }
1438
1439 NumIntermediates = NumVectorRegs;
1440
1441 MVT NewVT = MVT::getVectorVT(EltTy, EC);
1442 if (!TLI->isTypeLegal(NewVT))
1443 NewVT = EltTy;
1444 IntermediateVT = NewVT;
1445
1446 unsigned LaneSizeInBits = NewVT.getScalarSizeInBits();
1447
1448 // Convert sizes such as i33 to i64.
1449 LaneSizeInBits = llvm::bit_ceil(LaneSizeInBits);
1450
1451 MVT DestVT = TLI->getRegisterType(NewVT);
1452 RegisterVT = DestVT;
1453 if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
1454 return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits());
1455
1456 // Otherwise, promotion or legal types use the same number of registers as
1457 // the vector decimated to the appropriate level.
1458 return NumVectorRegs;
1459}
1460
1461/// isLegalRC - Return true if the value types that can be represented by the
1462/// specified register class are all legal.
1464 const TargetRegisterClass &RC) const {
1465 for (const auto *I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I)
1466 if (isTypeLegal(*I))
1467 return true;
1468 return false;
1469}
1470
1471/// Replace/modify any TargetFrameIndex operands with a targte-dependent
1472/// sequence of memory operands that is recognized by PrologEpilogInserter.
1475 MachineBasicBlock *MBB) const {
1476 MachineInstr *MI = &InitialMI;
1477 MachineFunction &MF = *MI->getMF();
1478 MachineFrameInfo &MFI = MF.getFrameInfo();
1479
1480 // We're handling multiple types of operands here:
1481 // PATCHPOINT MetaArgs - live-in, read only, direct
1482 // STATEPOINT Deopt Spill - live-through, read only, indirect
1483 // STATEPOINT Deopt Alloca - live-through, read only, direct
1484 // (We're currently conservative and mark the deopt slots read/write in
1485 // practice.)
1486 // STATEPOINT GC Spill - live-through, read/write, indirect
1487 // STATEPOINT GC Alloca - live-through, read/write, direct
1488 // The live-in vs live-through is handled already (the live through ones are
1489 // all stack slots), but we need to handle the different type of stackmap
1490 // operands and memory effects here.
1491
1492 if (llvm::none_of(MI->operands(),
1493 [](MachineOperand &Operand) { return Operand.isFI(); }))
1494 return MBB;
1495
1496 MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc());
1497
1498 // Inherit previous memory operands.
1499 MIB.cloneMemRefs(*MI);
1500
1501 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1502 MachineOperand &MO = MI->getOperand(i);
1503 if (!MO.isFI()) {
1504 // Index of Def operand this Use it tied to.
1505 // Since Defs are coming before Uses, if Use is tied, then
1506 // index of Def must be smaller that index of that Use.
1507 // Also, Defs preserve their position in new MI.
1508 unsigned TiedTo = i;
1509 if (MO.isReg() && MO.isTied())
1510 TiedTo = MI->findTiedOperandIdx(i);
1511 MIB.add(MO);
1512 if (TiedTo < i)
1513 MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1);
1514 continue;
1515 }
1516
1517 // foldMemoryOperand builds a new MI after replacing a single FI operand
1518 // with the canonical set of five x86 addressing-mode operands.
1519 int FI = MO.getIndex();
1520
1521 // Add frame index operands recognized by stackmaps.cpp
1523 // indirect-mem-ref tag, size, #FI, offset.
1524 // Used for spills inserted by StatepointLowering. This codepath is not
1525 // used for patchpoints/stackmaps at all, for these spilling is done via
1526 // foldMemoryOperand callback only.
1527 assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity");
1528 MIB.addImm(StackMaps::IndirectMemRefOp);
1529 MIB.addImm(MFI.getObjectSize(FI));
1530 MIB.add(MO);
1531 MIB.addImm(0);
1532 } else {
1533 // direct-mem-ref tag, #FI, offset.
1534 // Used by patchpoint, and direct alloca arguments to statepoints
1535 MIB.addImm(StackMaps::DirectMemRefOp);
1536 MIB.add(MO);
1537 MIB.addImm(0);
1538 }
1539
1540 assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!");
1541
1542 // Add a new memory operand for this FI.
1543 assert(MFI.getObjectOffset(FI) != -1);
1544
1545 // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and
1546 // PATCHPOINT should be updated to do the same. (TODO)
1547 if (MI->getOpcode() != TargetOpcode::STATEPOINT) {
1548 auto Flags = MachineMemOperand::MOLoad;
1550 MachinePointerInfo::getFixedStack(MF, FI), Flags,
1552 MIB->addMemOperand(MF, MMO);
1553 }
1554 }
1556 MI->eraseFromParent();
1557 return MBB;
1558}
1559
1560/// findRepresentativeClass - Return the largest legal super-reg register class
1561/// of the register class for the specified type and its associated "cost".
1562// This function is in TargetLowering because it uses RegClassForVT which would
1563// need to be moved to TargetRegisterInfo and would necessitate moving
1564// isTypeLegal over as well - a massive change that would just require
1565// TargetLowering having a TargetRegisterInfo class member that it would use.
1566std::pair<const TargetRegisterClass *, uint8_t>
1568 MVT VT) const {
1569 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
1570 if (!RC)
1571 return std::make_pair(RC, 0);
1572
1573 // Compute the set of all super-register classes.
1574 BitVector SuperRegRC(TRI->getNumRegClasses());
1575 for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI)
1576 SuperRegRC.setBitsInMask(RCI.getMask());
1577
1578 // Find the first legal register class with the largest spill size.
1579 const TargetRegisterClass *BestRC = RC;
1580 for (unsigned i : SuperRegRC.set_bits()) {
1581 const TargetRegisterClass *SuperRC = TRI->getRegClass(i);
1582 // We want the largest possible spill size.
1583 if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC))
1584 continue;
1585 if (!isLegalRC(*TRI, *SuperRC))
1586 continue;
1587 BestRC = SuperRC;
1588 }
1589 return std::make_pair(BestRC, 1);
1590}
1591
1592/// computeRegisterProperties - Once all of the register classes are added,
1593/// this allows us to compute derived properties we expose.
1595 const TargetRegisterInfo *TRI) {
1596 // Everything defaults to needing one register.
1597 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
1598 NumRegistersForVT[i] = 1;
1599 RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;
1600 }
1601 // ...except isVoid, which doesn't need any registers.
1602 NumRegistersForVT[MVT::isVoid] = 0;
1603
1604 // Find the largest integer register class.
1605 unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;
1606 for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg)
1607 assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
1608
1609 // Every integer value type larger than this largest register takes twice as
1610 // many registers to represent as the previous ValueType.
1611 for (unsigned ExpandedReg = LargestIntReg + 1;
1612 ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) {
1613 NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
1614 RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;
1615 TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);
1616 ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,
1618 }
1619
1620 // Inspect all of the ValueType's smaller than the largest integer
1621 // register to see which ones need promotion.
1622 unsigned LegalIntReg = LargestIntReg;
1623 for (unsigned IntReg = LargestIntReg - 1;
1624 IntReg >= (unsigned)MVT::i1; --IntReg) {
1625 MVT IVT = (MVT::SimpleValueType)IntReg;
1626 if (isTypeLegal(IVT)) {
1627 LegalIntReg = IntReg;
1628 } else {
1629 RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
1630 (MVT::SimpleValueType)LegalIntReg;
1631 ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
1632 }
1633 }
1634
1635 // ppcf128 type is really two f64's.
1636 if (!isTypeLegal(MVT::ppcf128)) {
1637 if (isTypeLegal(MVT::f64)) {
1638 NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];
1639 RegisterTypeForVT[MVT::ppcf128] = MVT::f64;
1640 TransformToType[MVT::ppcf128] = MVT::f64;
1641 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);
1642 } else {
1643 NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128];
1644 RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128];
1645 TransformToType[MVT::ppcf128] = MVT::i128;
1646 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat);
1647 }
1648 }
1649
1650 // Decide how to handle f128. If the target does not have native f128 support,
1651 // expand it to i128 and we will be generating soft float library calls.
1652 if (!isTypeLegal(MVT::f128)) {
1653 NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];
1654 RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];
1655 TransformToType[MVT::f128] = MVT::i128;
1656 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
1657 }
1658
1659 // Decide how to handle f80. If the target does not have native f80 support,
1660 // expand it to i96 and we will be generating soft float library calls.
1661 if (!isTypeLegal(MVT::f80)) {
1662 NumRegistersForVT[MVT::f80] = 3*NumRegistersForVT[MVT::i32];
1663 RegisterTypeForVT[MVT::f80] = RegisterTypeForVT[MVT::i32];
1664 TransformToType[MVT::f80] = MVT::i32;
1665 ValueTypeActions.setTypeAction(MVT::f80, TypeSoftenFloat);
1666 }
1667
1668 // Decide how to handle f64. If the target does not have native f64 support,
1669 // expand it to i64 and we will be generating soft float library calls.
1670 if (!isTypeLegal(MVT::f64)) {
1671 NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
1672 RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
1673 TransformToType[MVT::f64] = MVT::i64;
1674 ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);
1675 }
1676
1677 // Decide how to handle f32. If the target does not have native f32 support,
1678 // expand it to i32 and we will be generating soft float library calls.
1679 if (!isTypeLegal(MVT::f32)) {
1680 NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
1681 RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
1682 TransformToType[MVT::f32] = MVT::i32;
1683 ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);
1684 }
1685
1686 // Decide how to handle f16. If the target does not have native f16 support,
1687 // promote it to f32, because there are no f16 library calls (except for
1688 // conversions).
1689 if (!isTypeLegal(MVT::f16)) {
1690 // Allow targets to control how we legalize half.
1691 bool SoftPromoteHalfType = softPromoteHalfType();
1692 bool UseFPRegsForHalfType = !SoftPromoteHalfType || useFPRegsForHalfType();
1693
1694 if (!UseFPRegsForHalfType) {
1695 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16];
1696 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16];
1697 } else {
1698 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];
1699 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];
1700 }
1701 TransformToType[MVT::f16] = MVT::f32;
1702 if (SoftPromoteHalfType) {
1703 ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf);
1704 } else {
1705 ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);
1706 }
1707 }
1708
1709 // Decide how to handle bf16. If the target does not have native bf16 support,
1710 // promote it to f32, because there are no bf16 library calls (except for
1711 // converting from f32 to bf16).
1712 if (!isTypeLegal(MVT::bf16)) {
1713 NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];
1714 RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];
1715 TransformToType[MVT::bf16] = MVT::f32;
1716 ValueTypeActions.setTypeAction(MVT::bf16, TypeSoftPromoteHalf);
1717 }
1718
1719 // Loop over all of the vector value types to see which need transformations.
1720 for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
1721 i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
1722 MVT VT = (MVT::SimpleValueType) i;
1723 if (isTypeLegal(VT))
1724 continue;
1725
1726 MVT EltVT = VT.getVectorElementType();
1728 bool IsLegalWiderType = false;
1729 bool IsScalable = VT.isScalableVector();
1730 LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT);
1731 switch (PreferredAction) {
1732 case TypePromoteInteger: {
1733 MVT::SimpleValueType EndVT = IsScalable ?
1734 MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE :
1735 MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE;
1736 // Try to promote the elements of integer vectors. If no legal
1737 // promotion was found, fall through to the widen-vector method.
1738 for (unsigned nVT = i + 1;
1739 (MVT::SimpleValueType)nVT <= EndVT; ++nVT) {
1740 MVT SVT = (MVT::SimpleValueType) nVT;
1741 // Promote vectors of integers to vectors with the same number
1742 // of elements, with a wider element type.
1743 if (SVT.getScalarSizeInBits() > EltVT.getFixedSizeInBits() &&
1744 SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) {
1745 TransformToType[i] = SVT;
1746 RegisterTypeForVT[i] = SVT;
1747 NumRegistersForVT[i] = 1;
1748 ValueTypeActions.setTypeAction(VT, TypePromoteInteger);
1749 IsLegalWiderType = true;
1750 break;
1751 }
1752 }
1753 if (IsLegalWiderType)
1754 break;
1755 [[fallthrough]];
1756 }
1757
1758 case TypeWidenVector:
1759 if (isPowerOf2_32(EC.getKnownMinValue())) {
1760 // Try to widen the vector.
1761 for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
1762 MVT SVT = (MVT::SimpleValueType) nVT;
1763 if (SVT.getVectorElementType() == EltVT &&
1764 SVT.isScalableVector() == IsScalable &&
1766 EC.getKnownMinValue() &&
1767 isTypeLegal(SVT)) {
1768 TransformToType[i] = SVT;
1769 RegisterTypeForVT[i] = SVT;
1770 NumRegistersForVT[i] = 1;
1771 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1772 IsLegalWiderType = true;
1773 break;
1774 }
1775 }
1776 if (IsLegalWiderType)
1777 break;
1778 } else {
1779 // Only widen to the next power of 2 to keep consistency with EVT.
1780 MVT NVT = VT.getPow2VectorType();
1781 if (isTypeLegal(NVT)) {
1782 TransformToType[i] = NVT;
1783 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1784 RegisterTypeForVT[i] = NVT;
1785 NumRegistersForVT[i] = 1;
1786 break;
1787 }
1788 }
1789 [[fallthrough]];
1790
1791 case TypeSplitVector:
1792 case TypeScalarizeVector: {
1793 MVT IntermediateVT;
1794 MVT RegisterVT;
1795 unsigned NumIntermediates;
1796 unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT,
1797 NumIntermediates, RegisterVT, this);
1798 NumRegistersForVT[i] = NumRegisters;
1799 assert(NumRegistersForVT[i] == NumRegisters &&
1800 "NumRegistersForVT size cannot represent NumRegisters!");
1801 RegisterTypeForVT[i] = RegisterVT;
1802
1803 MVT NVT = VT.getPow2VectorType();
1804 if (NVT == VT) {
1805 // Type is already a power of 2. The default action is to split.
1806 TransformToType[i] = MVT::Other;
1807 if (PreferredAction == TypeScalarizeVector)
1808 ValueTypeActions.setTypeAction(VT, TypeScalarizeVector);
1809 else if (PreferredAction == TypeSplitVector)
1810 ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1811 else if (EC.getKnownMinValue() > 1)
1812 ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1813 else
1814 ValueTypeActions.setTypeAction(VT, EC.isScalable()
1817 } else {
1818 TransformToType[i] = NVT;
1819 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1820 }
1821 break;
1822 }
1823 default:
1824 llvm_unreachable("Unknown vector legalization action!");
1825 }
1826 }
1827
1828 // Determine the 'representative' register class for each value type.
1829 // An representative register class is the largest (meaning one which is
1830 // not a sub-register class / subreg register class) legal register class for
1831 // a group of value types. For example, on i386, i8, i16, and i32
1832 // representative would be GR32; while on x86_64 it's GR64.
1833 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
1834 const TargetRegisterClass* RRC;
1835 uint8_t Cost;
1837 RepRegClassForVT[i] = RRC;
1838 RepRegClassCostForVT[i] = Cost;
1839 }
1840}
1841
1843 EVT VT) const {
1844 assert(!VT.isVector() && "No default SetCC type for vectors!");
1845 return getPointerTy(DL).SimpleTy;
1846}
1847
1849 return MVT::i32; // return the default value
1850}
1851
1852/// getVectorTypeBreakdown - Vector types are broken down into some number of
1853/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
1854/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
1855/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
1856///
1857/// This method returns the number of registers needed, and the VT for each
1858/// register. It also returns the VT and quantity of the intermediate values
1859/// before they are promoted/expanded.
1861 EVT VT, EVT &IntermediateVT,
1862 unsigned &NumIntermediates,
1863 MVT &RegisterVT) const {
1864 ElementCount EltCnt = VT.getVectorElementCount();
1865
1866 // If there is a wider vector type with the same element type as this one,
1867 // or a promoted vector type that has the same number of elements which
1868 // are wider, then we should convert to that legal vector type.
1869 // This handles things like <2 x float> -> <4 x float> and
1870 // <4 x i1> -> <4 x i32>.
1871 LegalizeTypeAction TA = getTypeAction(Context, VT);
1872 if (!EltCnt.isScalar() &&
1873 (TA == TypeWidenVector || TA == TypePromoteInteger)) {
1874 EVT RegisterEVT = getTypeToTransformTo(Context, VT);
1875 if (isTypeLegal(RegisterEVT)) {
1876 IntermediateVT = RegisterEVT;
1877 RegisterVT = RegisterEVT.getSimpleVT();
1878 NumIntermediates = 1;
1879 return 1;
1880 }
1881 }
1882
1883 // Figure out the right, legal destination reg to copy into.
1884 EVT EltTy = VT.getVectorElementType();
1885
1886 unsigned NumVectorRegs = 1;
1887
1888 // Scalable vectors cannot be scalarized, so handle the legalisation of the
1889 // types like done elsewhere in SelectionDAG.
1890 if (EltCnt.isScalable()) {
1891 LegalizeKind LK;
1892 EVT PartVT = VT;
1893 do {
1894 // Iterate until we've found a legal (part) type to hold VT.
1895 LK = getTypeConversion(Context, PartVT);
1896 PartVT = LK.second;
1897 } while (LK.first != TypeLegal);
1898
1899 if (!PartVT.isVector()) {
1901 "Don't know how to legalize this scalable vector type");
1902 }
1903
1904 NumIntermediates =
1907 IntermediateVT = PartVT;
1908 RegisterVT = getRegisterType(Context, IntermediateVT);
1909 return NumIntermediates;
1910 }
1911
1912 // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally
1913 // we could break down into LHS/RHS like LegalizeDAG does.
1914 if (!isPowerOf2_32(EltCnt.getKnownMinValue())) {
1915 NumVectorRegs = EltCnt.getKnownMinValue();
1916 EltCnt = ElementCount::getFixed(1);
1917 }
1918
1919 // Divide the input until we get to a supported size. This will always
1920 // end with a scalar if the target doesn't support vectors.
1921 while (EltCnt.getKnownMinValue() > 1 &&
1922 !isTypeLegal(EVT::getVectorVT(Context, EltTy, EltCnt))) {
1923 EltCnt = EltCnt.divideCoefficientBy(2);
1924 NumVectorRegs <<= 1;
1925 }
1926
1927 NumIntermediates = NumVectorRegs;
1928
1929 EVT NewVT = EVT::getVectorVT(Context, EltTy, EltCnt);
1930 if (!isTypeLegal(NewVT))
1931 NewVT = EltTy;
1932 IntermediateVT = NewVT;
1933
1934 MVT DestVT = getRegisterType(Context, NewVT);
1935 RegisterVT = DestVT;
1936
1937 if (EVT(DestVT).bitsLT(NewVT)) { // Value is expanded, e.g. i64 -> i16.
1938 TypeSize NewVTSize = NewVT.getSizeInBits();
1939 // Convert sizes such as i33 to i64.
1940 if (!llvm::has_single_bit<uint32_t>(NewVTSize.getKnownMinValue()))
1941 NewVTSize = NewVTSize.coefficientNextPowerOf2();
1942 return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
1943 }
1944
1945 // Otherwise, promotion or legal types use the same number of registers as
1946 // the vector decimated to the appropriate level.
1947 return NumVectorRegs;
1948}
1949
1951 uint64_t NumCases,
1953 ProfileSummaryInfo *PSI,
1954 BlockFrequencyInfo *BFI) const {
1955 // FIXME: This function check the maximum table size and density, but the
1956 // minimum size is not checked. It would be nice if the minimum size is
1957 // also combined within this function. Currently, the minimum size check is
1958 // performed in findJumpTable() in SelectionDAGBuiler and
1959 // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
1960 const bool OptForSize =
1961 SI->getParent()->getParent()->hasOptSize() ||
1962 llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI);
1963 const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);
1964 const unsigned MaxJumpTableSize = getMaximumJumpTableSize();
1965
1966 // Check whether the number of cases is small enough and
1967 // the range is dense enough for a jump table.
1968 return (OptForSize || Range <= MaxJumpTableSize) &&
1969 (NumCases * 100 >= Range * MinDensity);
1970}
1971
1973 EVT ConditionVT) const {
1974 return getRegisterType(Context, ConditionVT);
1975}
1976
1977/// Get the EVTs and ArgFlags collections that represent the legalized return
1978/// type of the given function. This does not require a DAG or a return value,
1979/// and is suitable for use before any DAGs for the function are constructed.
1980/// TODO: Move this out of TargetLowering.cpp.
1982 AttributeList attr,
1984 const TargetLowering &TLI, const DataLayout &DL) {
1985 SmallVector<EVT, 4> ValueVTs;
1986 ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
1987 unsigned NumValues = ValueVTs.size();
1988 if (NumValues == 0) return;
1989
1990 for (unsigned j = 0, f = NumValues; j != f; ++j) {
1991 EVT VT = ValueVTs[j];
1992 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1993
1994 if (attr.hasRetAttr(Attribute::SExt))
1995 ExtendKind = ISD::SIGN_EXTEND;
1996 else if (attr.hasRetAttr(Attribute::ZExt))
1997 ExtendKind = ISD::ZERO_EXTEND;
1998
1999 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2000 VT = TLI.getTypeForExtReturn(ReturnType->getContext(), VT, ExtendKind);
2001
2002 unsigned NumParts =
2003 TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT);
2004 MVT PartVT =
2005 TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT);
2006
2007 // 'inreg' on function refers to return value
2009 if (attr.hasRetAttr(Attribute::InReg))
2010 Flags.setInReg();
2011
2012 // Propagate extension type if any
2013 if (attr.hasRetAttr(Attribute::SExt))
2014 Flags.setSExt();
2015 else if (attr.hasRetAttr(Attribute::ZExt))
2016 Flags.setZExt();
2017
2018 for (unsigned i = 0; i < NumParts; ++i) {
2019 ISD::ArgFlagsTy OutFlags = Flags;
2020 if (NumParts > 1 && i == 0)
2021 OutFlags.setSplit();
2022 else if (i == NumParts - 1 && i != 0)
2023 OutFlags.setSplitEnd();
2024
2025 Outs.push_back(
2026 ISD::OutputArg(OutFlags, PartVT, VT, /*isfixed=*/true, 0, 0));
2027 }
2028 }
2029}
2030
2031/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
2032/// function arguments in the caller parameter area. This is the actual
2033/// alignment, not its logarithm.
2035 const DataLayout &DL) const {
2036 return DL.getABITypeAlign(Ty).value();
2037}
2038
2040 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
2041 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
2042 // Check if the specified alignment is sufficient based on the data layout.
2043 // TODO: While using the data layout works in practice, a better solution
2044 // would be to implement this check directly (make this a virtual function).
2045 // For example, the ABI alignment may change based on software platform while
2046 // this function should only be affected by hardware implementation.
2047 Type *Ty = VT.getTypeForEVT(Context);
2048 if (VT.isZeroSized() || Alignment >= DL.getABITypeAlign(Ty)) {
2049 // Assume that an access that meets the ABI-specified alignment is fast.
2050 if (Fast != nullptr)
2051 *Fast = 1;
2052 return true;
2053 }
2054
2055 // This is a misaligned access.
2056 return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
2057}
2058
2060 LLVMContext &Context, const DataLayout &DL, EVT VT,
2061 const MachineMemOperand &MMO, unsigned *Fast) const {
2062 return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(),
2063 MMO.getAlign(), MMO.getFlags(), Fast);
2064}
2065
2067 const DataLayout &DL, EVT VT,
2068 unsigned AddrSpace, Align Alignment,
2070 unsigned *Fast) const {
2071 return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment,
2072 Flags, Fast);
2073}
2074
2076 const DataLayout &DL, EVT VT,
2077 const MachineMemOperand &MMO,
2078 unsigned *Fast) const {
2079 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
2080 MMO.getFlags(), Fast);
2081}
2082
2084 const DataLayout &DL, LLT Ty,
2085 const MachineMemOperand &MMO,
2086 unsigned *Fast) const {
2087 EVT VT = getApproximateEVTForLLT(Ty, DL, Context);
2088 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
2089 MMO.getFlags(), Fast);
2090}
2091
2092//===----------------------------------------------------------------------===//
2093// TargetTransformInfo Helpers
2094//===----------------------------------------------------------------------===//
2095
2097 enum InstructionOpcodes {
2098#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
2099#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
2100#include "llvm/IR/Instruction.def"
2101 };
2102 switch (static_cast<InstructionOpcodes>(Opcode)) {
2103 case Ret: return 0;
2104 case Br: return 0;
2105 case Switch: return 0;
2106 case IndirectBr: return 0;
2107 case Invoke: return 0;
2108 case CallBr: return 0;
2109 case Resume: return 0;
2110 case Unreachable: return 0;
2111 case CleanupRet: return 0;
2112 case CatchRet: return 0;
2113 case CatchPad: return 0;
2114 case CatchSwitch: return 0;
2115 case CleanupPad: return 0;
2116 case FNeg: return ISD::FNEG;
2117 case Add: return ISD::ADD;
2118 case FAdd: return ISD::FADD;
2119 case Sub: return ISD::SUB;
2120 case FSub: return ISD::FSUB;
2121 case Mul: return ISD::MUL;
2122 case FMul: return ISD::FMUL;
2123 case UDiv: return ISD::UDIV;
2124 case SDiv: return ISD::SDIV;
2125 case FDiv: return ISD::FDIV;
2126 case URem: return ISD::UREM;
2127 case SRem: return ISD::SREM;
2128 case FRem: return ISD::FREM;
2129 case Shl: return ISD::SHL;
2130 case LShr: return ISD::SRL;
2131 case AShr: return ISD::SRA;
2132 case And: return ISD::AND;
2133 case Or: return ISD::OR;
2134 case Xor: return ISD::XOR;
2135 case Alloca: return 0;
2136 case Load: return ISD::LOAD;
2137 case Store: return ISD::STORE;
2138 case GetElementPtr: return 0;
2139 case Fence: return 0;
2140 case AtomicCmpXchg: return 0;
2141 case AtomicRMW: return 0;
2142 case Trunc: return ISD::TRUNCATE;
2143 case ZExt: return ISD::ZERO_EXTEND;
2144 case SExt: return ISD::SIGN_EXTEND;
2145 case FPToUI: return ISD::FP_TO_UINT;
2146 case FPToSI: return ISD::FP_TO_SINT;
2147 case UIToFP: return ISD::UINT_TO_FP;
2148 case SIToFP: return ISD::SINT_TO_FP;
2149 case FPTrunc: return ISD::FP_ROUND;
2150 case FPExt: return ISD::FP_EXTEND;
2151 case PtrToInt: return ISD::BITCAST;
2152 case IntToPtr: return ISD::BITCAST;
2153 case BitCast: return ISD::BITCAST;
2154 case AddrSpaceCast: return ISD::ADDRSPACECAST;
2155 case ICmp: return ISD::SETCC;
2156 case FCmp: return ISD::SETCC;
2157 case PHI: return 0;
2158 case Call: return 0;
2159 case Select: return ISD::SELECT;
2160 case UserOp1: return 0;
2161 case UserOp2: return 0;
2162 case VAArg: return 0;
2163 case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
2164 case InsertElement: return ISD::INSERT_VECTOR_ELT;
2165 case ShuffleVector: return ISD::VECTOR_SHUFFLE;
2166 case ExtractValue: return ISD::MERGE_VALUES;
2167 case InsertValue: return ISD::MERGE_VALUES;
2168 case LandingPad: return 0;
2169 case Freeze: return ISD::FREEZE;
2170 }
2171
2172 llvm_unreachable("Unknown instruction type encountered!");
2173}
2174
2175Value *
2177 bool UseTLS) const {
2178 // compiler-rt provides a variable with a magic name. Targets that do not
2179 // link with compiler-rt may also provide such a variable.
2180 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
2181 const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";
2182 auto UnsafeStackPtr =
2183 dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar));
2184
2185 Type *StackPtrTy = PointerType::getUnqual(M->getContext());
2186
2187 if (!UnsafeStackPtr) {
2188 auto TLSModel = UseTLS ?
2191 // The global variable is not defined yet, define it ourselves.
2192 // We use the initial-exec TLS model because we do not support the
2193 // variable living anywhere other than in the main executable.
2194 UnsafeStackPtr = new GlobalVariable(
2195 *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr,
2196 UnsafeStackPtrVar, nullptr, TLSModel);
2197 } else {
2198 // The variable exists, check its type and attributes.
2199 if (UnsafeStackPtr->getValueType() != StackPtrTy)
2200 report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type");
2201 if (UseTLS != UnsafeStackPtr->isThreadLocal())
2202 report_fatal_error(Twine(UnsafeStackPtrVar) + " must " +
2203 (UseTLS ? "" : "not ") + "be thread-local");
2204 }
2205 return UnsafeStackPtr;
2206}
2207
2208Value *
2210 if (!TM.getTargetTriple().isAndroid())
2211 return getDefaultSafeStackPointerLocation(IRB, true);
2212
2213 // Android provides a libc function to retrieve the address of the current
2214 // thread's unsafe stack pointer.
2215 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
2216 auto *PtrTy = PointerType::getUnqual(M->getContext());
2217 FunctionCallee Fn =
2218 M->getOrInsertFunction("__safestack_pointer_address", PtrTy);
2219 return IRB.CreateCall(Fn);
2220}
2221
2222//===----------------------------------------------------------------------===//
2223// Loop Strength Reduction hooks
2224//===----------------------------------------------------------------------===//
2225
2226/// isLegalAddressingMode - Return true if the addressing mode represented
2227/// by AM is legal for this target, for a load/store of the specified type.
2229 const AddrMode &AM, Type *Ty,
2230 unsigned AS, Instruction *I) const {
2231 // The default implementation of this implements a conservative RISCy, r+r and
2232 // r+i addr mode.
2233
2234 // Scalable offsets not supported
2235 if (AM.ScalableOffset)
2236 return false;
2237
2238 // Allows a sign-extended 16-bit immediate field.
2239 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
2240 return false;
2241
2242 // No global is ever allowed as a base.
2243 if (AM.BaseGV)
2244 return false;
2245
2246 // Only support r+r,
2247 switch (AM.Scale) {
2248 case 0: // "r+i" or just "i", depending on HasBaseReg.
2249 break;
2250 case 1:
2251 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
2252 return false;
2253 // Otherwise we have r+r or r+i.
2254 break;
2255 case 2:
2256 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
2257 return false;
2258 // Allow 2*r as r+r.
2259 break;
2260 default: // Don't allow n * r
2261 return false;
2262 }
2263
2264 return true;
2265}
2266
2267//===----------------------------------------------------------------------===//
2268// Stack Protector
2269//===----------------------------------------------------------------------===//
2270
2271// For OpenBSD return its special guard variable. Otherwise return nullptr,
2272// so that SelectionDAG handle SSP.
2274 if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
2275 Module &M = *IRB.GetInsertBlock()->getParent()->getParent();
2276 PointerType *PtrTy = PointerType::getUnqual(M.getContext());
2277 Constant *C = M.getOrInsertGlobal("__guard_local", PtrTy);
2278 if (GlobalVariable *G = dyn_cast_or_null<GlobalVariable>(C))
2279 G->setVisibility(GlobalValue::HiddenVisibility);
2280 return C;
2281 }
2282 return nullptr;
2283}
2284
2285// Currently only support "standard" __stack_chk_guard.
2286// TODO: add LOAD_STACK_GUARD support.
2288 if (!M.getNamedValue("__stack_chk_guard")) {
2289 auto *GV = new GlobalVariable(M, PointerType::getUnqual(M.getContext()),
2291 nullptr, "__stack_chk_guard");
2292
2293 // FreeBSD has "__stack_chk_guard" defined externally on libc.so
2294 if (M.getDirectAccessExternalData() &&
2296 !(TM.getTargetTriple().isPPC64() &&
2297 TM.getTargetTriple().isOSFreeBSD()) &&
2298 (!TM.getTargetTriple().isOSDarwin() ||
2300 GV->setDSOLocal(true);
2301 }
2302}
2303
2304// Currently only support "standard" __stack_chk_guard.
2305// TODO: add LOAD_STACK_GUARD support.
2307 return M.getNamedValue("__stack_chk_guard");
2308}
2309
2311 return nullptr;
2312}
2313
2316}
2317
2320}
2321
2322unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const {
2323 return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity;
2324}
2325
2327 return MaximumJumpTableSize;
2328}
2329
2332}
2333
2336}
2337
2339 if (TM.Options.LoopAlignment)
2340 return Align(TM.Options.LoopAlignment);
2341 return PrefLoopAlignment;
2342}
2343
2345 MachineBasicBlock *MBB) const {
2346 return MaxBytesForAlignment;
2347}
2348
2349//===----------------------------------------------------------------------===//
2350// Reciprocal Estimates
2351//===----------------------------------------------------------------------===//
2352
2353/// Get the reciprocal estimate attribute string for a function that will
2354/// override the target defaults.
2356 const Function &F = MF.getFunction();
2357 return F.getFnAttribute("reciprocal-estimates").getValueAsString();
2358}
2359
2360/// Construct a string for the given reciprocal operation of the given type.
2361/// This string should match the corresponding option to the front-end's
2362/// "-mrecip" flag assuming those strings have been passed through in an
2363/// attribute string. For example, "vec-divf" for a division of a vXf32.
2364static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
2365 std::string Name = VT.isVector() ? "vec-" : "";
2366
2367 Name += IsSqrt ? "sqrt" : "div";
2368
2369 // TODO: Handle other float types?
2370 if (VT.getScalarType() == MVT::f64) {
2371 Name += "d";
2372 } else if (VT.getScalarType() == MVT::f16) {
2373 Name += "h";
2374 } else {
2375 assert(VT.getScalarType() == MVT::f32 &&
2376 "Unexpected FP type for reciprocal estimate");
2377 Name += "f";
2378 }
2379
2380 return Name;
2381}
2382
2383/// Return the character position and value (a single numeric character) of a
2384/// customized refinement operation in the input string if it exists. Return
2385/// false if there is no customized refinement step count.
2386static bool parseRefinementStep(StringRef In, size_t &Position,
2387 uint8_t &Value) {
2388 const char RefStepToken = ':';
2389 Position = In.find(RefStepToken);
2390 if (Position == StringRef::npos)
2391 return false;
2392
2393 StringRef RefStepString = In.substr(Position + 1);
2394 // Allow exactly one numeric character for the additional refinement
2395 // step parameter.
2396 if (RefStepString.size() == 1) {
2397 char RefStepChar = RefStepString[0];
2398 if (isDigit(RefStepChar)) {
2399 Value = RefStepChar - '0';
2400 return true;
2401 }
2402 }
2403 report_fatal_error("Invalid refinement step for -recip.");
2404}
2405
2406/// For the input attribute string, return one of the ReciprocalEstimate enum
2407/// status values (enabled, disabled, or not specified) for this operation on
2408/// the specified data type.
2409static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
2410 if (Override.empty())
2412
2413 SmallVector<StringRef, 4> OverrideVector;
2414 Override.split(OverrideVector, ',');
2415 unsigned NumArgs = OverrideVector.size();
2416
2417 // Check if "all", "none", or "default" was specified.
2418 if (NumArgs == 1) {
2419 // Look for an optional setting of the number of refinement steps needed
2420 // for this type of reciprocal operation.
2421 size_t RefPos;
2422 uint8_t RefSteps;
2423 if (parseRefinementStep(Override, RefPos, RefSteps)) {
2424 // Split the string for further processing.
2425 Override = Override.substr(0, RefPos);
2426 }
2427
2428 // All reciprocal types are enabled.
2429 if (Override == "all")
2431
2432 // All reciprocal types are disabled.
2433 if (Override == "none")
2435
2436 // Target defaults for enablement are used.
2437 if (Override == "default")
2439 }
2440
2441 // The attribute string may omit the size suffix ('f'/'d').
2442 std::string VTName = getReciprocalOpName(IsSqrt, VT);
2443 std::string VTNameNoSize = VTName;
2444 VTNameNoSize.pop_back();
2445 static const char DisabledPrefix = '!';
2446
2447 for (StringRef RecipType : OverrideVector) {
2448 size_t RefPos;
2449 uint8_t RefSteps;
2450 if (parseRefinementStep(RecipType, RefPos, RefSteps))
2451 RecipType = RecipType.substr(0, RefPos);
2452
2453 // Ignore the disablement token for string matching.
2454 bool IsDisabled = RecipType[0] == DisabledPrefix;
2455 if (IsDisabled)
2456 RecipType = RecipType.substr(1);
2457
2458 if (RecipType == VTName || RecipType == VTNameNoSize)
2461 }
2462
2464}
2465
2466/// For the input attribute string, return the customized refinement step count
2467/// for this operation on the specified data type. If the step count does not
2468/// exist, return the ReciprocalEstimate enum value for unspecified.
2469static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
2470 if (Override.empty())
2472
2473 SmallVector<StringRef, 4> OverrideVector;
2474 Override.split(OverrideVector, ',');
2475 unsigned NumArgs = OverrideVector.size();
2476
2477 // Check if "all", "default", or "none" was specified.
2478 if (NumArgs == 1) {
2479 // Look for an optional setting of the number of refinement steps needed
2480 // for this type of reciprocal operation.
2481 size_t RefPos;
2482 uint8_t RefSteps;
2483 if (!parseRefinementStep(Override, RefPos, RefSteps))
2485
2486 // Split the string for further processing.
2487 Override = Override.substr(0, RefPos);
2488 assert(Override != "none" &&
2489 "Disabled reciprocals, but specifed refinement steps?");
2490
2491 // If this is a general override, return the specified number of steps.
2492 if (Override == "all" || Override == "default")
2493 return RefSteps;
2494 }
2495
2496 // The attribute string may omit the size suffix ('f'/'d').
2497 std::string VTName = getReciprocalOpName(IsSqrt, VT);
2498 std::string VTNameNoSize = VTName;
2499 VTNameNoSize.pop_back();
2500
2501 for (StringRef RecipType : OverrideVector) {
2502 size_t RefPos;
2503 uint8_t RefSteps;
2504 if (!parseRefinementStep(RecipType, RefPos, RefSteps))
2505 continue;
2506
2507 RecipType = RecipType.substr(0, RefPos);
2508 if (RecipType == VTName || RecipType == VTNameNoSize)
2509 return RefSteps;
2510 }
2511
2513}
2514
2516 MachineFunction &MF) const {
2517 return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
2518}
2519
2521 MachineFunction &MF) const {
2522 return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
2523}
2524
2526 MachineFunction &MF) const {
2527 return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
2528}
2529
2531 MachineFunction &MF) const {
2532 return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
2533}
2534
2536 EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG,
2537 const MachineMemOperand &MMO) const {
2538 // Single-element vectors are scalarized, so we should generally avoid having
2539 // any memory operations on such types, as they would get scalarized too.
2540 if (LoadVT.isFixedLengthVector() && BitcastVT.isFixedLengthVector() &&
2541 BitcastVT.getVectorNumElements() == 1)
2542 return false;
2543
2544 // Don't do if we could do an indexed load on the original type, but not on
2545 // the new one.
2546 if (!LoadVT.isSimple() || !BitcastVT.isSimple())
2547 return true;
2548
2549 MVT LoadMVT = LoadVT.getSimpleVT();
2550
2551 // Don't bother doing this if it's just going to be promoted again later, as
2552 // doing so might interfere with other combines.
2553 if (getOperationAction(ISD::LOAD, LoadMVT) == Promote &&
2554 getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())
2555 return false;
2556
2557 unsigned Fast = 0;
2558 return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,
2559 MMO, &Fast) &&
2560 Fast;
2561}
2562
2565}
2566
2568 const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC,
2569 const TargetLibraryInfo *LibInfo) const {
2571 if (LI.isVolatile())
2573
2574 if (LI.hasMetadata(LLVMContext::MD_nontemporal))
2576
2577 if (LI.hasMetadata(LLVMContext::MD_invariant_load))
2579
2581 LI.getAlign(), DL, &LI, AC,
2582 /*DT=*/nullptr, LibInfo))
2584
2585 Flags |= getTargetMMOFlags(LI);
2586 return Flags;
2587}
2588
2591 const DataLayout &DL) const {
2593
2594 if (SI.isVolatile())
2596
2597 if (SI.hasMetadata(LLVMContext::MD_nontemporal))
2599
2600 // FIXME: Not preserving dereferenceable
2601 Flags |= getTargetMMOFlags(SI);
2602 return Flags;
2603}
2604
2607 const DataLayout &DL) const {
2609
2610 if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) {
2611 if (RMW->isVolatile())
2613 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) {
2614 if (CmpX->isVolatile())
2616 } else
2617 llvm_unreachable("not an atomic instruction");
2618
2619 // FIXME: Not preserving dereferenceable
2620 Flags |= getTargetMMOFlags(AI);
2621 return Flags;
2622}
2623
2625 Instruction *Inst,
2626 AtomicOrdering Ord) const {
2627 if (isReleaseOrStronger(Ord) && Inst->hasAtomicStore())
2628 return Builder.CreateFence(Ord);
2629 else
2630 return nullptr;
2631}
2632
2634 Instruction *Inst,
2635 AtomicOrdering Ord) const {
2636 if (isAcquireOrStronger(Ord))
2637 return Builder.CreateFence(Ord);
2638 else
2639 return nullptr;
2640}
2641
2642//===----------------------------------------------------------------------===//
2643// GlobalISel Hooks
2644//===----------------------------------------------------------------------===//
2645
2647 const TargetTransformInfo *TTI) const {
2648 auto &MF = *MI.getMF();
2649 auto &MRI = MF.getRegInfo();
2650 // Assuming a spill and reload of a value has a cost of 1 instruction each,
2651 // this helper function computes the maximum number of uses we should consider
2652 // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We
2653 // break even in terms of code size when the original MI has 2 users vs
2654 // choosing to potentially spill. Any more than 2 users we we have a net code
2655 // size increase. This doesn't take into account register pressure though.
2656 auto maxUses = [](unsigned RematCost) {
2657 // A cost of 1 means remats are basically free.
2658 if (RematCost == 1)
2659 return std::numeric_limits<unsigned>::max();
2660 if (RematCost == 2)
2661 return 2U;
2662
2663 // Remat is too expensive, only sink if there's one user.
2664 if (RematCost > 2)
2665 return 1U;
2666 llvm_unreachable("Unexpected remat cost");
2667 };
2668
2669 switch (MI.getOpcode()) {
2670 default:
2671 return false;
2672 // Constants-like instructions should be close to their users.
2673 // We don't want long live-ranges for them.
2674 case TargetOpcode::G_CONSTANT:
2675 case TargetOpcode::G_FCONSTANT:
2676 case TargetOpcode::G_FRAME_INDEX:
2677 case TargetOpcode::G_INTTOPTR:
2678 return true;
2679 case TargetOpcode::G_GLOBAL_VALUE: {
2680 unsigned RematCost = TTI->getGISelRematGlobalCost();
2681 Register Reg = MI.getOperand(0).getReg();
2682 unsigned MaxUses = maxUses(RematCost);
2683 if (MaxUses == UINT_MAX)
2684 return true; // Remats are "free" so always localize.
2685 return MRI.hasAtMostUserInstrs(Reg, MaxUses);
2686 }
2687 }
2688}
unsigned const MachineRegisterInfo * MRI
amdgpu AMDGPU Register Bank Select
Rewrite undef for PHI
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
return RetTy
std::string Name
IRTranslator LLVM IR MI
#define LCALL5(A)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
const char LLVMTargetMachineRef TM
static bool isDigit(const char C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
static bool darwinHasSinCos(const Triple &TT)
static cl::opt< bool > JumpIsExpensiveOverride("jump-is-expensive", cl::init(false), cl::desc("Do not create extra branches to split comparison logic."), cl::Hidden)
#define OP_TO_LIBCALL(Name, Enum)
static cl::opt< unsigned > MinimumJumpTableEntries("min-jump-table-entries", cl::init(4), cl::Hidden, cl::desc("Set minimum number of entries to use a jump table."))
static cl::opt< bool > DisableStrictNodeMutation("disable-strictnode-mutation", cl::desc("Don't mutate strict-float node to a legalize node"), cl::init(false), cl::Hidden)
static bool parseRefinementStep(StringRef In, size_t &Position, uint8_t &Value)
Return the character position and value (a single numeric character) of a customized refinement opera...
static cl::opt< unsigned > MaximumJumpTableSize("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden, cl::desc("Set maximum size of jump tables."))
static cl::opt< unsigned > JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden, cl::desc("Minimum density for building a jump table in " "a normal function"))
Minimum jump table density for normal functions.
static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT, TargetLoweringBase *TLI)
static std::string getReciprocalOpName(bool IsSqrt, EVT VT)
Construct a string for the given reciprocal operation of the given type.
static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override)
For the input attribute string, return the customized refinement step count for this operation on the...
static void InitCmpLibcallCCs(ISD::CondCode *CCs)
InitCmpLibcallCCs - Set default comparison libcall CC.
static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override)
For the input attribute string, return one of the ReciprocalEstimate enum status values (enabled,...
static StringRef getRecipEstimateForFunc(MachineFunction &MF)
Get the reciprocal estimate attribute string for a function that will override the target defaults.
static cl::opt< unsigned > OptsizeJumpTableDensity("optsize-jump-table-density", cl::init(40), cl::Hidden, cl::desc("Minimum density for building a jump table in " "an optsize function"))
Minimum jump table density for -Os or -Oz functions.
This file describes how to lower LLVM code to machine code.
This pass exposes codegen information to IR-level passes.
Class for arbitrary precision integers.
Definition: APInt.h:78
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:495
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:696
bool hasRetAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the return value.
Definition: Attributes.h:820
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:209
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Definition: BitVector.h:707
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:140
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
This class represents a range of values.
Definition: ConstantRange.h:47
unsigned getActiveBits() const
Compute the maximal number of active bits needed to represent every value in this range.
ConstantRange umul_sat(const ConstantRange &Other) const
Perform an unsigned saturating multiplication of two constant ranges.
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
This is an important base class in LLVM.
Definition: Constant.h:42
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size in bytes, rounded up to a whole number of bytes.
Definition: DataLayout.cpp:750
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition: TypeSize.h:314
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:311
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:322
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:68
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:91
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Definition: IRBuilder.h:1839
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:171
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2417
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:363
@ MAX_INT_BITS
Maximum number of bits that can be specified.
Definition: DerivedTypes.h:52
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:174
Value * getPointerOperand()
Definition: Instructions.h:253
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:203
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:209
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static auto all_valuetypes()
SimpleValueType Iteration.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
MVT getPow2VectorType() const
Widens the length of the given vector MVT up to the nearest power of 2 and returns that type.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool isStatepointSpillSlotObjectIndex(int ObjectIdx) const
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:572
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
A description of a memory reference used in the backend.
unsigned getAddrSpace() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
void freezeReservedRegs()
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Class to represent pointers.
Definition: DerivedTypes.h:646
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
Analysis providing profile information.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:227
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:486
LLVMContext * getContext() const
Definition: SelectionDAG.h:499
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:290
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:693
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:564
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
static constexpr size_t npos
Definition: StringRef.h:52
bool isValid() const
Returns true if this iterator is still pointing at a valid entry.
Multiway switch.
Provides information about what library functions are available for the current target.
LegalizeTypeAction getTypeAction(MVT VT) const
void setTypeAction(MVT VT, LegalizeTypeAction Action)
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void initActions()
Initialize all of the actions to default values.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const
Returns the target-specific address of the unsafe stack pointer.
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const
Check whether or not MI needs to be moved close to its uses.
virtual unsigned getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const
Return the maximum amount of bytes allowed to be emitted when padding for alignment.
void setMaximumJumpTableSize(unsigned)
Indicate the maximum number of entries in jump tables.
virtual unsigned getMinimumJumpTableEntries() const
Return lower limit for number of blocks in a jump table.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases, uint64_t Range, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) const
Return true if lowering to a jump table is suitable for a set of case clusters which may contain NumC...
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual bool useFPRegsForHalfType() const
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
virtual bool softPromoteHalfType() const
unsigned getMaximumJumpTableSize() const
Return upper limit for number of entries in a jump table.
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC, bool ZeroIsPoison, const ConstantRange *VScaleRange) const
Return the minimum number of bits required to hold the maximum possible number of trailing zero vecto...
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setAtomicLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Let target indicate that an extending atomic load of the specified type is legal.
Value * getDefaultSafeStackPointerLocation(IRBuilderBase &IRB, bool UseTLS) const
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual Align getPrefLoopAlignment(MachineLoop *ML=nullptr) const
Return the preferred loop alignment.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
virtual MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const
Returns preferred type for switch condition.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual bool isJumpTableRelative() const
virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const
Return the type to use for a scalar shift opcode, given the shifted amount type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
unsigned getMinimumJumpTableDensity(bool OptForSize) const
Return lower limit of the density in a jump table.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
TargetLoweringBase(const TargetMachine &TM)
NOTE: The TargetMachine owns TLOF.
LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const
Return pair that represents the legalization kind (first) that needs to happen to EVT (second) in ord...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should continue looking for chain dependencies when trying to find a more...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
virtual Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
virtual Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
Inserts in the IR a target-specific intrinsic specifying a fence.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
std::pair< LegalizeTypeAction, EVT > LegalizeKind
LegalizeKind holds the legalization kind that needs to happen to EVT in order to type-legalize it.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
bool isPositionIndependent() const
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
Reloc::Model getRelocationModel() const
Returns the code generation relocation model.
TargetOptions Options
unsigned LoopAlignment
If greater than 0, override TargetLoweringBase::PrefLoopAlignment.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
unsigned getGISelRematGlobalCost() const
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isWindowsGNUEnvironment() const
Definition: Triple.h:655
bool isAndroid() const
Tests whether the target is Android.
Definition: Triple.h:767
@ aarch64_32
Definition: Triple.h:53
bool isOSFreeBSD() const
Definition: Triple.h:582
bool isPPC64() const
Tests whether the target is 64-bit PowerPC (little and big endian).
Definition: Triple.h:962
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Definition: Triple.h:556
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
constexpr LeafTy coefficientNextPowerOf2() const
Definition: TypeSize.h:262
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition: TypeSize.h:254
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
Definition: CallingConv.h:114
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:770
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:243
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:743
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:1031
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1372
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1403
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:511
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1302
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:374
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1304
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1305
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:276
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1035
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:501
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:246
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1064
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:380
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1054
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:803
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1287
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:810
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:557
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1388
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1392
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:708
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1058
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:840
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1402
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1300
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:923
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1301
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:966
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:387
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1443
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:794
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:676
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1221
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1385
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:742
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1254
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1389
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:778
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:950
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:334
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1303
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:521
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:356
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:747
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1404
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:641
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:330
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1397
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1049
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1026
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:725
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1299
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:614
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1298
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:1011
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:549
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:800
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1244
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:876
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1281
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1306
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:998
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1248
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:366
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:338
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:829
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:818
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:688
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:393
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:908
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:310
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1405
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1296
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:1017
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1297
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:856
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1215
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:700
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1241
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1040
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition: ISDOpcodes.h:696
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:671
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1386
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:286
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:538
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:626
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1295
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:971
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:889
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:851
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:927
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:875
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1393
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:806
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1373
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:507
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:347
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1353
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1045
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:320
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1566
static const int LAST_INDEXED_MODE
Definition: ISDOpcodes.h:1517
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getOutlineAtomicHelper(const Libcall(&LC)[5][4], AtomicOrdering Order, uint64_t MemSize)
Return the outline atomics value for the given atomic ordering, access size and set of libcalls for a...
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:353
EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx)
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
Definition: Sequence.h:337
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
Definition: Loads.cpp:201
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
constexpr force_iteration_on_noniterable_enum_t force_iteration_on_noniterable_enum
Definition: Sequence.h:108
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:342
bool isReleaseOrStronger(AtomicOrdering AO)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:291
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1736
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition: MathExtras.h:403
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ FMul
Product of floats.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
@ FAdd
Sum of floats.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
bool isAcquireOrStronger(AtomicOrdering AO)
InstructionCost Cost
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
EVT getPow2VectorType(LLVMContext &Context) const
Widens the length of the given vector EVT up to the nearest power of 2 and returns that type.
Definition: ValueTypes.h:462
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition: ValueTypes.h:455
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
Definition: ValueTypes.h:404
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:203
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool isZeroSized() const
Test if the given EVT has zero size, this will fail if called on a scalable type.
Definition: ValueTypes.h:131
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...