LLVM 18.0.0git
TargetLoweringBase.cpp
Go to the documentation of this file.
1//===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLoweringBase class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/BitVector.h"
14#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/Loads.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/CallingConv.h"
40#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/Function.h"
43#include "llvm/IR/GlobalValue.h"
45#include "llvm/IR/IRBuilder.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/Type.h"
57#include <algorithm>
58#include <cassert>
59#include <cstdint>
60#include <cstring>
61#include <iterator>
62#include <string>
63#include <tuple>
64#include <utility>
65
66using namespace llvm;
67
69 "jump-is-expensive", cl::init(false),
70 cl::desc("Do not create extra branches to split comparison logic."),
72
74 ("min-jump-table-entries", cl::init(4), cl::Hidden,
75 cl::desc("Set minimum number of entries to use a jump table."));
76
78 ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden,
79 cl::desc("Set maximum size of jump tables."));
80
81/// Minimum jump table density for normal functions.
83 JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
84 cl::desc("Minimum density for building a jump table in "
85 "a normal function"));
86
87/// Minimum jump table density for -Os or -Oz functions.
89 "optsize-jump-table-density", cl::init(40), cl::Hidden,
90 cl::desc("Minimum density for building a jump table in "
91 "an optsize function"));
92
93// FIXME: This option is only to test if the strict fp operation processed
94// correctly by preventing mutating strict fp operation to normal fp operation
95// during development. When the backend supports strict float operation, this
96// option will be meaningless.
97static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation",
98 cl::desc("Don't mutate strict-float node to a legalize node"),
99 cl::init(false), cl::Hidden);
100
101static bool darwinHasSinCos(const Triple &TT) {
102 assert(TT.isOSDarwin() && "should be called with darwin triple");
103 // Don't bother with 32 bit x86.
104 if (TT.getArch() == Triple::x86)
105 return false;
106 // Macos < 10.9 has no sincos_stret.
107 if (TT.isMacOSX())
108 return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit();
109 // iOS < 7.0 has no sincos_stret.
110 if (TT.isiOS())
111 return !TT.isOSVersionLT(7, 0);
112 // Any other darwin such as WatchOS/TvOS is new enough.
113 return true;
114}
115
116void TargetLoweringBase::InitLibcalls(const Triple &TT) {
117#define HANDLE_LIBCALL(code, name) \
118 setLibcallName(RTLIB::code, name);
119#include "llvm/IR/RuntimeLibcalls.def"
120#undef HANDLE_LIBCALL
121 // Initialize calling conventions to their default.
122 for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
124
125 // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
126 if (TT.isPPC()) {
127 setLibcallName(RTLIB::ADD_F128, "__addkf3");
128 setLibcallName(RTLIB::SUB_F128, "__subkf3");
129 setLibcallName(RTLIB::MUL_F128, "__mulkf3");
130 setLibcallName(RTLIB::DIV_F128, "__divkf3");
131 setLibcallName(RTLIB::POWI_F128, "__powikf2");
132 setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
133 setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
134 setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
135 setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
136 setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
137 setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
138 setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
139 setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
140 setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
141 setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
142 setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
143 setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
144 setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
145 setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
146 setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
147 setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
148 setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
149 setLibcallName(RTLIB::UNE_F128, "__nekf2");
150 setLibcallName(RTLIB::OGE_F128, "__gekf2");
151 setLibcallName(RTLIB::OLT_F128, "__ltkf2");
152 setLibcallName(RTLIB::OLE_F128, "__lekf2");
153 setLibcallName(RTLIB::OGT_F128, "__gtkf2");
154 setLibcallName(RTLIB::UO_F128, "__unordkf2");
155 }
156
157 // A few names are different on particular architectures or environments.
158 if (TT.isOSDarwin()) {
159 // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
160 // of the gnueabi-style __gnu_*_ieee.
161 // FIXME: What about other targets?
162 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
163 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
164
165 // Some darwins have an optimized __bzero/bzero function.
166 switch (TT.getArch()) {
167 case Triple::x86:
168 case Triple::x86_64:
169 if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
170 setLibcallName(RTLIB::BZERO, "__bzero");
171 break;
172 case Triple::aarch64:
174 setLibcallName(RTLIB::BZERO, "bzero");
175 break;
176 default:
177 break;
178 }
179
180 if (darwinHasSinCos(TT)) {
181 setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
182 setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
183 if (TT.isWatchABI()) {
184 setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
186 setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
188 }
189 }
190 } else {
191 setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
192 setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
193 }
194
195 if (TT.isGNUEnvironment() || TT.isOSFuchsia() ||
196 (TT.isAndroid() && !TT.isAndroidVersionLT(9))) {
197 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
198 setLibcallName(RTLIB::SINCOS_F64, "sincos");
199 setLibcallName(RTLIB::SINCOS_F80, "sincosl");
200 setLibcallName(RTLIB::SINCOS_F128, "sincosl");
201 setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
202 }
203
204 if (TT.isPS()) {
205 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
206 setLibcallName(RTLIB::SINCOS_F64, "sincos");
207 }
208
209 if (TT.isOSOpenBSD()) {
210 setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
211 }
212
213 if (TT.isOSWindows() && !TT.isOSCygMing()) {
214 setLibcallName(RTLIB::LDEXP_F32, nullptr);
215 setLibcallName(RTLIB::LDEXP_F80, nullptr);
216 setLibcallName(RTLIB::LDEXP_F128, nullptr);
217 setLibcallName(RTLIB::LDEXP_PPCF128, nullptr);
218
219 setLibcallName(RTLIB::FREXP_F32, nullptr);
220 setLibcallName(RTLIB::FREXP_F80, nullptr);
221 setLibcallName(RTLIB::FREXP_F128, nullptr);
222 setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
223 }
224}
225
226/// GetFPLibCall - Helper to return the right libcall for the given floating
227/// point type, or UNKNOWN_LIBCALL if there is none.
229 RTLIB::Libcall Call_F32,
230 RTLIB::Libcall Call_F64,
231 RTLIB::Libcall Call_F80,
232 RTLIB::Libcall Call_F128,
233 RTLIB::Libcall Call_PPCF128) {
234 return
235 VT == MVT::f32 ? Call_F32 :
236 VT == MVT::f64 ? Call_F64 :
237 VT == MVT::f80 ? Call_F80 :
238 VT == MVT::f128 ? Call_F128 :
239 VT == MVT::ppcf128 ? Call_PPCF128 :
240 RTLIB::UNKNOWN_LIBCALL;
241}
242
243/// getFPEXT - Return the FPEXT_*_* value for the given types, or
244/// UNKNOWN_LIBCALL if there is none.
246 if (OpVT == MVT::f16) {
247 if (RetVT == MVT::f32)
248 return FPEXT_F16_F32;
249 if (RetVT == MVT::f64)
250 return FPEXT_F16_F64;
251 if (RetVT == MVT::f80)
252 return FPEXT_F16_F80;
253 if (RetVT == MVT::f128)
254 return FPEXT_F16_F128;
255 } else if (OpVT == MVT::f32) {
256 if (RetVT == MVT::f64)
257 return FPEXT_F32_F64;
258 if (RetVT == MVT::f128)
259 return FPEXT_F32_F128;
260 if (RetVT == MVT::ppcf128)
261 return FPEXT_F32_PPCF128;
262 } else if (OpVT == MVT::f64) {
263 if (RetVT == MVT::f128)
264 return FPEXT_F64_F128;
265 else if (RetVT == MVT::ppcf128)
266 return FPEXT_F64_PPCF128;
267 } else if (OpVT == MVT::f80) {
268 if (RetVT == MVT::f128)
269 return FPEXT_F80_F128;
270 }
271
272 return UNKNOWN_LIBCALL;
273}
274
275/// getFPROUND - Return the FPROUND_*_* value for the given types, or
276/// UNKNOWN_LIBCALL if there is none.
278 if (RetVT == MVT::f16) {
279 if (OpVT == MVT::f32)
280 return FPROUND_F32_F16;
281 if (OpVT == MVT::f64)
282 return FPROUND_F64_F16;
283 if (OpVT == MVT::f80)
284 return FPROUND_F80_F16;
285 if (OpVT == MVT::f128)
286 return FPROUND_F128_F16;
287 if (OpVT == MVT::ppcf128)
288 return FPROUND_PPCF128_F16;
289 } else if (RetVT == MVT::bf16) {
290 if (OpVT == MVT::f32)
291 return FPROUND_F32_BF16;
292 if (OpVT == MVT::f64)
293 return FPROUND_F64_BF16;
294 } else if (RetVT == MVT::f32) {
295 if (OpVT == MVT::f64)
296 return FPROUND_F64_F32;
297 if (OpVT == MVT::f80)
298 return FPROUND_F80_F32;
299 if (OpVT == MVT::f128)
300 return FPROUND_F128_F32;
301 if (OpVT == MVT::ppcf128)
302 return FPROUND_PPCF128_F32;
303 } else if (RetVT == MVT::f64) {
304 if (OpVT == MVT::f80)
305 return FPROUND_F80_F64;
306 if (OpVT == MVT::f128)
307 return FPROUND_F128_F64;
308 if (OpVT == MVT::ppcf128)
309 return FPROUND_PPCF128_F64;
310 } else if (RetVT == MVT::f80) {
311 if (OpVT == MVT::f128)
312 return FPROUND_F128_F80;
313 }
314
315 return UNKNOWN_LIBCALL;
316}
317
318/// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
319/// UNKNOWN_LIBCALL if there is none.
321 if (OpVT == MVT::f16) {
322 if (RetVT == MVT::i32)
323 return FPTOSINT_F16_I32;
324 if (RetVT == MVT::i64)
325 return FPTOSINT_F16_I64;
326 if (RetVT == MVT::i128)
327 return FPTOSINT_F16_I128;
328 } else if (OpVT == MVT::f32) {
329 if (RetVT == MVT::i32)
330 return FPTOSINT_F32_I32;
331 if (RetVT == MVT::i64)
332 return FPTOSINT_F32_I64;
333 if (RetVT == MVT::i128)
334 return FPTOSINT_F32_I128;
335 } else if (OpVT == MVT::f64) {
336 if (RetVT == MVT::i32)
337 return FPTOSINT_F64_I32;
338 if (RetVT == MVT::i64)
339 return FPTOSINT_F64_I64;
340 if (RetVT == MVT::i128)
341 return FPTOSINT_F64_I128;
342 } else if (OpVT == MVT::f80) {
343 if (RetVT == MVT::i32)
344 return FPTOSINT_F80_I32;
345 if (RetVT == MVT::i64)
346 return FPTOSINT_F80_I64;
347 if (RetVT == MVT::i128)
348 return FPTOSINT_F80_I128;
349 } else if (OpVT == MVT::f128) {
350 if (RetVT == MVT::i32)
351 return FPTOSINT_F128_I32;
352 if (RetVT == MVT::i64)
353 return FPTOSINT_F128_I64;
354 if (RetVT == MVT::i128)
355 return FPTOSINT_F128_I128;
356 } else if (OpVT == MVT::ppcf128) {
357 if (RetVT == MVT::i32)
358 return FPTOSINT_PPCF128_I32;
359 if (RetVT == MVT::i64)
360 return FPTOSINT_PPCF128_I64;
361 if (RetVT == MVT::i128)
362 return FPTOSINT_PPCF128_I128;
363 }
364 return UNKNOWN_LIBCALL;
365}
366
367/// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
368/// UNKNOWN_LIBCALL if there is none.
370 if (OpVT == MVT::f16) {
371 if (RetVT == MVT::i32)
372 return FPTOUINT_F16_I32;
373 if (RetVT == MVT::i64)
374 return FPTOUINT_F16_I64;
375 if (RetVT == MVT::i128)
376 return FPTOUINT_F16_I128;
377 } else if (OpVT == MVT::f32) {
378 if (RetVT == MVT::i32)
379 return FPTOUINT_F32_I32;
380 if (RetVT == MVT::i64)
381 return FPTOUINT_F32_I64;
382 if (RetVT == MVT::i128)
383 return FPTOUINT_F32_I128;
384 } else if (OpVT == MVT::f64) {
385 if (RetVT == MVT::i32)
386 return FPTOUINT_F64_I32;
387 if (RetVT == MVT::i64)
388 return FPTOUINT_F64_I64;
389 if (RetVT == MVT::i128)
390 return FPTOUINT_F64_I128;
391 } else if (OpVT == MVT::f80) {
392 if (RetVT == MVT::i32)
393 return FPTOUINT_F80_I32;
394 if (RetVT == MVT::i64)
395 return FPTOUINT_F80_I64;
396 if (RetVT == MVT::i128)
397 return FPTOUINT_F80_I128;
398 } else if (OpVT == MVT::f128) {
399 if (RetVT == MVT::i32)
400 return FPTOUINT_F128_I32;
401 if (RetVT == MVT::i64)
402 return FPTOUINT_F128_I64;
403 if (RetVT == MVT::i128)
404 return FPTOUINT_F128_I128;
405 } else if (OpVT == MVT::ppcf128) {
406 if (RetVT == MVT::i32)
407 return FPTOUINT_PPCF128_I32;
408 if (RetVT == MVT::i64)
409 return FPTOUINT_PPCF128_I64;
410 if (RetVT == MVT::i128)
411 return FPTOUINT_PPCF128_I128;
412 }
413 return UNKNOWN_LIBCALL;
414}
415
416/// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
417/// UNKNOWN_LIBCALL if there is none.
419 if (OpVT == MVT::i32) {
420 if (RetVT == MVT::f16)
421 return SINTTOFP_I32_F16;
422 if (RetVT == MVT::f32)
423 return SINTTOFP_I32_F32;
424 if (RetVT == MVT::f64)
425 return SINTTOFP_I32_F64;
426 if (RetVT == MVT::f80)
427 return SINTTOFP_I32_F80;
428 if (RetVT == MVT::f128)
429 return SINTTOFP_I32_F128;
430 if (RetVT == MVT::ppcf128)
431 return SINTTOFP_I32_PPCF128;
432 } else if (OpVT == MVT::i64) {
433 if (RetVT == MVT::f16)
434 return SINTTOFP_I64_F16;
435 if (RetVT == MVT::f32)
436 return SINTTOFP_I64_F32;
437 if (RetVT == MVT::f64)
438 return SINTTOFP_I64_F64;
439 if (RetVT == MVT::f80)
440 return SINTTOFP_I64_F80;
441 if (RetVT == MVT::f128)
442 return SINTTOFP_I64_F128;
443 if (RetVT == MVT::ppcf128)
444 return SINTTOFP_I64_PPCF128;
445 } else if (OpVT == MVT::i128) {
446 if (RetVT == MVT::f16)
447 return SINTTOFP_I128_F16;
448 if (RetVT == MVT::f32)
449 return SINTTOFP_I128_F32;
450 if (RetVT == MVT::f64)
451 return SINTTOFP_I128_F64;
452 if (RetVT == MVT::f80)
453 return SINTTOFP_I128_F80;
454 if (RetVT == MVT::f128)
455 return SINTTOFP_I128_F128;
456 if (RetVT == MVT::ppcf128)
457 return SINTTOFP_I128_PPCF128;
458 }
459 return UNKNOWN_LIBCALL;
460}
461
462/// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
463/// UNKNOWN_LIBCALL if there is none.
465 if (OpVT == MVT::i32) {
466 if (RetVT == MVT::f16)
467 return UINTTOFP_I32_F16;
468 if (RetVT == MVT::f32)
469 return UINTTOFP_I32_F32;
470 if (RetVT == MVT::f64)
471 return UINTTOFP_I32_F64;
472 if (RetVT == MVT::f80)
473 return UINTTOFP_I32_F80;
474 if (RetVT == MVT::f128)
475 return UINTTOFP_I32_F128;
476 if (RetVT == MVT::ppcf128)
477 return UINTTOFP_I32_PPCF128;
478 } else if (OpVT == MVT::i64) {
479 if (RetVT == MVT::f16)
480 return UINTTOFP_I64_F16;
481 if (RetVT == MVT::f32)
482 return UINTTOFP_I64_F32;
483 if (RetVT == MVT::f64)
484 return UINTTOFP_I64_F64;
485 if (RetVT == MVT::f80)
486 return UINTTOFP_I64_F80;
487 if (RetVT == MVT::f128)
488 return UINTTOFP_I64_F128;
489 if (RetVT == MVT::ppcf128)
490 return UINTTOFP_I64_PPCF128;
491 } else if (OpVT == MVT::i128) {
492 if (RetVT == MVT::f16)
493 return UINTTOFP_I128_F16;
494 if (RetVT == MVT::f32)
495 return UINTTOFP_I128_F32;
496 if (RetVT == MVT::f64)
497 return UINTTOFP_I128_F64;
498 if (RetVT == MVT::f80)
499 return UINTTOFP_I128_F80;
500 if (RetVT == MVT::f128)
501 return UINTTOFP_I128_F128;
502 if (RetVT == MVT::ppcf128)
503 return UINTTOFP_I128_PPCF128;
504 }
505 return UNKNOWN_LIBCALL;
506}
507
509 return getFPLibCall(RetVT, POWI_F32, POWI_F64, POWI_F80, POWI_F128,
510 POWI_PPCF128);
511}
512
514 return getFPLibCall(RetVT, LDEXP_F32, LDEXP_F64, LDEXP_F80, LDEXP_F128,
515 LDEXP_PPCF128);
516}
517
519 return getFPLibCall(RetVT, FREXP_F32, FREXP_F64, FREXP_F80, FREXP_F128,
520 FREXP_PPCF128);
521}
522
524 MVT VT) {
525 unsigned ModeN, ModelN;
526 switch (VT.SimpleTy) {
527 case MVT::i8:
528 ModeN = 0;
529 break;
530 case MVT::i16:
531 ModeN = 1;
532 break;
533 case MVT::i32:
534 ModeN = 2;
535 break;
536 case MVT::i64:
537 ModeN = 3;
538 break;
539 case MVT::i128:
540 ModeN = 4;
541 break;
542 default:
543 return UNKNOWN_LIBCALL;
544 }
545
546 switch (Order) {
548 ModelN = 0;
549 break;
551 ModelN = 1;
552 break;
554 ModelN = 2;
555 break;
558 ModelN = 3;
559 break;
560 default:
561 return UNKNOWN_LIBCALL;
562 }
563
564#define LCALLS(A, B) \
565 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
566#define LCALL5(A) \
567 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
568 switch (Opc) {
570 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};
571 return LC[ModeN][ModelN];
572 }
573 case ISD::ATOMIC_SWAP: {
574 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};
575 return LC[ModeN][ModelN];
576 }
578 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};
579 return LC[ModeN][ModelN];
580 }
581 case ISD::ATOMIC_LOAD_OR: {
582 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};
583 return LC[ModeN][ModelN];
584 }
586 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};
587 return LC[ModeN][ModelN];
588 }
590 const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};
591 return LC[ModeN][ModelN];
592 }
593 default:
594 return UNKNOWN_LIBCALL;
595 }
596#undef LCALLS
597#undef LCALL5
598}
599
601#define OP_TO_LIBCALL(Name, Enum) \
602 case Name: \
603 switch (VT.SimpleTy) { \
604 default: \
605 return UNKNOWN_LIBCALL; \
606 case MVT::i8: \
607 return Enum##_1; \
608 case MVT::i16: \
609 return Enum##_2; \
610 case MVT::i32: \
611 return Enum##_4; \
612 case MVT::i64: \
613 return Enum##_8; \
614 case MVT::i128: \
615 return Enum##_16; \
616 }
617
618 switch (Opc) {
619 OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET)
620 OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP)
621 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD)
622 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB)
623 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND)
624 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR)
625 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR)
626 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND)
627 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX)
628 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX)
629 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN)
630 OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN)
631 }
632
633#undef OP_TO_LIBCALL
634
635 return UNKNOWN_LIBCALL;
636}
637
639 switch (ElementSize) {
640 case 1:
641 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1;
642 case 2:
643 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2;
644 case 4:
645 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4;
646 case 8:
647 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8;
648 case 16:
649 return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16;
650 default:
651 return UNKNOWN_LIBCALL;
652 }
653}
654
656 switch (ElementSize) {
657 case 1:
658 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;
659 case 2:
660 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;
661 case 4:
662 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;
663 case 8:
664 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;
665 case 16:
666 return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;
667 default:
668 return UNKNOWN_LIBCALL;
669 }
670}
671
673 switch (ElementSize) {
674 case 1:
675 return MEMSET_ELEMENT_UNORDERED_ATOMIC_1;
676 case 2:
677 return MEMSET_ELEMENT_UNORDERED_ATOMIC_2;
678 case 4:
679 return MEMSET_ELEMENT_UNORDERED_ATOMIC_4;
680 case 8:
681 return MEMSET_ELEMENT_UNORDERED_ATOMIC_8;
682 case 16:
683 return MEMSET_ELEMENT_UNORDERED_ATOMIC_16;
684 default:
685 return UNKNOWN_LIBCALL;
686 }
687}
688
689/// InitCmpLibcallCCs - Set default comparison libcall CC.
691 std::fill(CCs, CCs + RTLIB::UNKNOWN_LIBCALL, ISD::SETCC_INVALID);
692 CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
693 CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
694 CCs[RTLIB::OEQ_F128] = ISD::SETEQ;
695 CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
696 CCs[RTLIB::UNE_F32] = ISD::SETNE;
697 CCs[RTLIB::UNE_F64] = ISD::SETNE;
698 CCs[RTLIB::UNE_F128] = ISD::SETNE;
699 CCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
700 CCs[RTLIB::OGE_F32] = ISD::SETGE;
701 CCs[RTLIB::OGE_F64] = ISD::SETGE;
702 CCs[RTLIB::OGE_F128] = ISD::SETGE;
703 CCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
704 CCs[RTLIB::OLT_F32] = ISD::SETLT;
705 CCs[RTLIB::OLT_F64] = ISD::SETLT;
706 CCs[RTLIB::OLT_F128] = ISD::SETLT;
707 CCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
708 CCs[RTLIB::OLE_F32] = ISD::SETLE;
709 CCs[RTLIB::OLE_F64] = ISD::SETLE;
710 CCs[RTLIB::OLE_F128] = ISD::SETLE;
711 CCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
712 CCs[RTLIB::OGT_F32] = ISD::SETGT;
713 CCs[RTLIB::OGT_F64] = ISD::SETGT;
714 CCs[RTLIB::OGT_F128] = ISD::SETGT;
715 CCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
716 CCs[RTLIB::UO_F32] = ISD::SETNE;
717 CCs[RTLIB::UO_F64] = ISD::SETNE;
718 CCs[RTLIB::UO_F128] = ISD::SETNE;
719 CCs[RTLIB::UO_PPCF128] = ISD::SETNE;
720}
721
722/// NOTE: The TargetMachine owns TLOF.
724 initActions();
725
726 // Perform these initializations only once.
732 HasMultipleConditionRegisters = false;
733 HasExtractBitsInsn = false;
734 JumpIsExpensive = JumpIsExpensiveOverride;
736 EnableExtLdPromotion = false;
737 StackPointerRegisterToSaveRestore = 0;
738 BooleanContents = UndefinedBooleanContent;
739 BooleanFloatContents = UndefinedBooleanContent;
740 BooleanVectorContents = UndefinedBooleanContent;
741 SchedPreferenceInfo = Sched::ILP;
744 MaxBytesForAlignment = 0;
745 // TODO: the default will be switched to 0 in the next commit, along
746 // with the Target-specific changes necessary.
747 MaxAtomicSizeInBitsSupported = 1024;
748
749 // Assume that even with libcalls, no target supports wider than 128 bit
750 // division.
751 MaxDivRemBitWidthSupported = 128;
752
753 MaxLargeFPConvertBitWidthSupported = llvm::IntegerType::MAX_INT_BITS;
754
755 MinCmpXchgSizeInBits = 0;
756 SupportsUnalignedAtomics = false;
757
758 std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
759
760 InitLibcalls(TM.getTargetTriple());
761 InitCmpLibcallCCs(CmpLibcallCCs);
762}
763
765 // All operations default to being supported.
766 memset(OpActions, 0, sizeof(OpActions));
767 memset(LoadExtActions, 0, sizeof(LoadExtActions));
768 memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
769 memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
770 memset(CondCodeActions, 0, sizeof(CondCodeActions));
771 std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
772 std::fill(std::begin(TargetDAGCombineArray),
773 std::end(TargetDAGCombineArray), 0);
774
775 // We're somewhat special casing MVT::i2 and MVT::i4. Ideally we want to
776 // remove this and targets should individually set these types if not legal.
779 for (MVT VT : {MVT::i2, MVT::i4})
780 OpActions[(unsigned)VT.SimpleTy][NT] = Expand;
781 }
782 for (MVT AVT : MVT::all_valuetypes()) {
783 for (MVT VT : {MVT::i2, MVT::i4, MVT::v128i2, MVT::v64i4}) {
784 setTruncStoreAction(AVT, VT, Expand);
787 }
788 }
789 for (unsigned IM = (unsigned)ISD::PRE_INC;
790 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
791 for (MVT VT : {MVT::i2, MVT::i4}) {
796 }
797 }
798
799 for (MVT VT : MVT::fp_valuetypes()) {
800 MVT IntVT = MVT::getIntegerVT(VT.getFixedSizeInBits());
801 if (IntVT.isValid()) {
804 }
805 }
806
807 // Set default actions for various operations.
808 for (MVT VT : MVT::all_valuetypes()) {
809 // Default all indexed load / store to expand.
810 for (unsigned IM = (unsigned)ISD::PRE_INC;
811 IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
816 }
817
818 // Most backends expect to see the node which just returns the value loaded.
820
821 // These operations default to expand.
839 VT, Expand);
840
841 // Overflow operations default to expand
844 VT, Expand);
845
846 // Carry-using overflow operations default to expand.
849 VT, Expand);
850
851 // ADDC/ADDE/SUBC/SUBE default to expand.
853 Expand);
854
855 // Halving adds
858 Expand);
859
860 // Absolute difference
862
863 // These default to Expand so they will be expanded to CTLZ/CTTZ by default.
865 Expand);
866
868
869 // These library functions default to expand.
872 VT, Expand);
873
874 // These operations default to expand for vector types.
875 if (VT.isVector())
880 VT, Expand);
881
882 // Constrained floating-point operations default to expand.
883#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
884 setOperationAction(ISD::STRICT_##DAGN, VT, Expand);
885#include "llvm/IR/ConstrainedOps.def"
886
887 // For most targets @llvm.get.dynamic.area.offset just returns 0.
889
890 // Vector reduction default to expand.
898 VT, Expand);
899
900 // Named vector shuffles default to expand.
902
903 // VP operations default to expand.
904#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
905 setOperationAction(ISD::SDOPC, VT, Expand);
906#include "llvm/IR/VPIntrinsics.def"
907
908 // FP environment operations default to expand.
912 }
913
914 // Most targets ignore the @llvm.prefetch intrinsic.
916
917 // Most targets also ignore the @llvm.readcyclecounter intrinsic.
919
920 // ConstantFP nodes default to expand. Targets can either change this to
921 // Legal, in which case all fp constants are legal, or use isFPImmLegal()
922 // to optimize expansions for certain constants.
924 {MVT::bf16, MVT::f16, MVT::f32, MVT::f64, MVT::f80, MVT::f128},
925 Expand);
926
927 // These library functions default to expand.
932 {MVT::f32, MVT::f64, MVT::f128}, Expand);
933
934 // Default ISD::TRAP to expand (which turns it into abort).
935 setOperationAction(ISD::TRAP, MVT::Other, Expand);
936
937 // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
938 // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
940
942
945
946 for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
949 }
951}
952
954 EVT) const {
955 return MVT::getIntegerVT(DL.getPointerSizeInBits(0));
956}
957
959 bool LegalTypes) const {
960 assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
961 if (LHSTy.isVector())
962 return LHSTy;
963 MVT ShiftVT =
964 LegalTypes ? getScalarShiftAmountTy(DL, LHSTy) : getPointerTy(DL);
965 // If any possible shift value won't fit in the prefered type, just use
966 // something safe. Assume it will be legalized when the shift is expanded.
967 if (ShiftVT.getSizeInBits() < Log2_32_Ceil(LHSTy.getSizeInBits()))
968 ShiftVT = MVT::i32;
969 assert(ShiftVT.getSizeInBits() >= Log2_32_Ceil(LHSTy.getSizeInBits()) &&
970 "ShiftVT is still too small!");
971 return ShiftVT;
972}
973
974bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
975 assert(isTypeLegal(VT));
976 switch (Op) {
977 default:
978 return false;
979 case ISD::SDIV:
980 case ISD::UDIV:
981 case ISD::SREM:
982 case ISD::UREM:
983 return true;
984 }
985}
986
988 unsigned DestAS) const {
989 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
990}
991
993 // If the command-line option was specified, ignore this request.
994 if (!JumpIsExpensiveOverride.getNumOccurrences())
995 JumpIsExpensive = isExpensive;
996}
997
1000 // If this is a simple type, use the ComputeRegisterProp mechanism.
1001 if (VT.isSimple()) {
1002 MVT SVT = VT.getSimpleVT();
1003 assert((unsigned)SVT.SimpleTy < std::size(TransformToType));
1004 MVT NVT = TransformToType[SVT.SimpleTy];
1005 LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);
1006
1007 assert((LA == TypeLegal || LA == TypeSoftenFloat ||
1008 LA == TypeSoftPromoteHalf ||
1009 (NVT.isVector() ||
1010 ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger)) &&
1011 "Promote may not follow Expand or Promote");
1012
1013 if (LA == TypeSplitVector)
1014 return LegalizeKind(LA, EVT(SVT).getHalfNumVectorElementsVT(Context));
1015 if (LA == TypeScalarizeVector)
1016 return LegalizeKind(LA, SVT.getVectorElementType());
1017 return LegalizeKind(LA, NVT);
1018 }
1019
1020 // Handle Extended Scalar Types.
1021 if (!VT.isVector()) {
1022 assert(VT.isInteger() && "Float types must be simple");
1023 unsigned BitSize = VT.getSizeInBits();
1024 // First promote to a power-of-two size, then expand if necessary.
1025 if (BitSize < 8 || !isPowerOf2_32(BitSize)) {
1027 assert(NVT != VT && "Unable to round integer VT");
1028 LegalizeKind NextStep = getTypeConversion(Context, NVT);
1029 // Avoid multi-step promotion.
1030 if (NextStep.first == TypePromoteInteger)
1031 return NextStep;
1032 // Return rounded integer type.
1033 return LegalizeKind(TypePromoteInteger, NVT);
1034 }
1035
1038 }
1039
1040 // Handle vector types.
1041 ElementCount NumElts = VT.getVectorElementCount();
1042 EVT EltVT = VT.getVectorElementType();
1043
1044 // Vectors with only one element are always scalarized.
1045 if (NumElts.isScalar())
1046 return LegalizeKind(TypeScalarizeVector, EltVT);
1047
1048 // Try to widen vector elements until the element type is a power of two and
1049 // promote it to a legal type later on, for example:
1050 // <3 x i8> -> <4 x i8> -> <4 x i32>
1051 if (EltVT.isInteger()) {
1052 // Vectors with a number of elements that is not a power of two are always
1053 // widened, for example <3 x i8> -> <4 x i8>.
1054 if (!VT.isPow2VectorType()) {
1055 NumElts = NumElts.coefficientNextPowerOf2();
1056 EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts);
1057 return LegalizeKind(TypeWidenVector, NVT);
1058 }
1059
1060 // Examine the element type.
1062
1063 // If type is to be expanded, split the vector.
1064 // <4 x i140> -> <2 x i140>
1065 if (LK.first == TypeExpandInteger) {
1070 }
1071
1072 // Promote the integer element types until a legal vector type is found
1073 // or until the element integer type is too big. If a legal type was not
1074 // found, fallback to the usual mechanism of widening/splitting the
1075 // vector.
1076 EVT OldEltVT = EltVT;
1077 while (true) {
1078 // Increase the bitwidth of the element to the next pow-of-two
1079 // (which is greater than 8 bits).
1080 EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits())
1082
1083 // Stop trying when getting a non-simple element type.
1084 // Note that vector elements may be greater than legal vector element
1085 // types. Example: X86 XMM registers hold 64bit element on 32bit
1086 // systems.
1087 if (!EltVT.isSimple())
1088 break;
1089
1090 // Build a new vector type and check if it is legal.
1091 MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
1092 // Found a legal promoted vector type.
1093 if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal)
1095 EVT::getVectorVT(Context, EltVT, NumElts));
1096 }
1097
1098 // Reset the type to the unexpanded type if we did not find a legal vector
1099 // type with a promoted vector element type.
1100 EltVT = OldEltVT;
1101 }
1102
1103 // Try to widen the vector until a legal type is found.
1104 // If there is no wider legal type, split the vector.
1105 while (true) {
1106 // Round up to the next power of 2.
1107 NumElts = NumElts.coefficientNextPowerOf2();
1108
1109 // If there is no simple vector type with this many elements then there
1110 // cannot be a larger legal vector type. Note that this assumes that
1111 // there are no skipped intermediate vector types in the simple types.
1112 if (!EltVT.isSimple())
1113 break;
1114 MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
1115 if (LargerVector == MVT())
1116 break;
1117
1118 // If this type is legal then widen the vector.
1119 if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal)
1120 return LegalizeKind(TypeWidenVector, LargerVector);
1121 }
1122
1123 // Widen odd vectors to next power of two.
1124 if (!VT.isPow2VectorType()) {
1125 EVT NVT = VT.getPow2VectorType(Context);
1126 return LegalizeKind(TypeWidenVector, NVT);
1127 }
1128
1131
1132 // Vectors with illegal element types are expanded.
1133 EVT NVT = EVT::getVectorVT(Context, EltVT,
1135 return LegalizeKind(TypeSplitVector, NVT);
1136}
1137
1138static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
1139 unsigned &NumIntermediates,
1140 MVT &RegisterVT,
1141 TargetLoweringBase *TLI) {
1142 // Figure out the right, legal destination reg to copy into.
1144 MVT EltTy = VT.getVectorElementType();
1145
1146 unsigned NumVectorRegs = 1;
1147
1148 // Scalable vectors cannot be scalarized, so splitting or widening is
1149 // required.
1150 if (VT.isScalableVector() && !isPowerOf2_32(EC.getKnownMinValue()))
1152 "Splitting or widening of non-power-of-2 MVTs is not implemented.");
1153
1154 // FIXME: We don't support non-power-of-2-sized vectors for now.
1155 // Ideally we could break down into LHS/RHS like LegalizeDAG does.
1156 if (!isPowerOf2_32(EC.getKnownMinValue())) {
1157 // Split EC to unit size (scalable property is preserved).
1158 NumVectorRegs = EC.getKnownMinValue();
1159 EC = ElementCount::getFixed(1);
1160 }
1161
1162 // Divide the input until we get to a supported size. This will
1163 // always end up with an EC that represent a scalar or a scalable
1164 // scalar.
1165 while (EC.getKnownMinValue() > 1 &&
1166 !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) {
1167 EC = EC.divideCoefficientBy(2);
1168 NumVectorRegs <<= 1;
1169 }
1170
1171 NumIntermediates = NumVectorRegs;
1172
1173 MVT NewVT = MVT::getVectorVT(EltTy, EC);
1174 if (!TLI->isTypeLegal(NewVT))
1175 NewVT = EltTy;
1176 IntermediateVT = NewVT;
1177
1178 unsigned LaneSizeInBits = NewVT.getScalarSizeInBits();
1179
1180 // Convert sizes such as i33 to i64.
1181 LaneSizeInBits = llvm::bit_ceil(LaneSizeInBits);
1182
1183 MVT DestVT = TLI->getRegisterType(NewVT);
1184 RegisterVT = DestVT;
1185 if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
1186 return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits());
1187
1188 // Otherwise, promotion or legal types use the same number of registers as
1189 // the vector decimated to the appropriate level.
1190 return NumVectorRegs;
1191}
1192
1193/// isLegalRC - Return true if the value types that can be represented by the
1194/// specified register class are all legal.
1196 const TargetRegisterClass &RC) const {
1197 for (const auto *I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I)
1198 if (isTypeLegal(*I))
1199 return true;
1200 return false;
1201}
1202
1203/// Replace/modify any TargetFrameIndex operands with a targte-dependent
1204/// sequence of memory operands that is recognized by PrologEpilogInserter.
1207 MachineBasicBlock *MBB) const {
1208 MachineInstr *MI = &InitialMI;
1209 MachineFunction &MF = *MI->getMF();
1210 MachineFrameInfo &MFI = MF.getFrameInfo();
1211
1212 // We're handling multiple types of operands here:
1213 // PATCHPOINT MetaArgs - live-in, read only, direct
1214 // STATEPOINT Deopt Spill - live-through, read only, indirect
1215 // STATEPOINT Deopt Alloca - live-through, read only, direct
1216 // (We're currently conservative and mark the deopt slots read/write in
1217 // practice.)
1218 // STATEPOINT GC Spill - live-through, read/write, indirect
1219 // STATEPOINT GC Alloca - live-through, read/write, direct
1220 // The live-in vs live-through is handled already (the live through ones are
1221 // all stack slots), but we need to handle the different type of stackmap
1222 // operands and memory effects here.
1223
1224 if (llvm::none_of(MI->operands(),
1225 [](MachineOperand &Operand) { return Operand.isFI(); }))
1226 return MBB;
1227
1228 MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc());
1229
1230 // Inherit previous memory operands.
1231 MIB.cloneMemRefs(*MI);
1232
1233 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1234 MachineOperand &MO = MI->getOperand(i);
1235 if (!MO.isFI()) {
1236 // Index of Def operand this Use it tied to.
1237 // Since Defs are coming before Uses, if Use is tied, then
1238 // index of Def must be smaller that index of that Use.
1239 // Also, Defs preserve their position in new MI.
1240 unsigned TiedTo = i;
1241 if (MO.isReg() && MO.isTied())
1242 TiedTo = MI->findTiedOperandIdx(i);
1243 MIB.add(MO);
1244 if (TiedTo < i)
1245 MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1);
1246 continue;
1247 }
1248
1249 // foldMemoryOperand builds a new MI after replacing a single FI operand
1250 // with the canonical set of five x86 addressing-mode operands.
1251 int FI = MO.getIndex();
1252
1253 // Add frame index operands recognized by stackmaps.cpp
1255 // indirect-mem-ref tag, size, #FI, offset.
1256 // Used for spills inserted by StatepointLowering. This codepath is not
1257 // used for patchpoints/stackmaps at all, for these spilling is done via
1258 // foldMemoryOperand callback only.
1259 assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity");
1260 MIB.addImm(StackMaps::IndirectMemRefOp);
1261 MIB.addImm(MFI.getObjectSize(FI));
1262 MIB.add(MO);
1263 MIB.addImm(0);
1264 } else {
1265 // direct-mem-ref tag, #FI, offset.
1266 // Used by patchpoint, and direct alloca arguments to statepoints
1267 MIB.addImm(StackMaps::DirectMemRefOp);
1268 MIB.add(MO);
1269 MIB.addImm(0);
1270 }
1271
1272 assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!");
1273
1274 // Add a new memory operand for this FI.
1275 assert(MFI.getObjectOffset(FI) != -1);
1276
1277 // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and
1278 // PATCHPOINT should be updated to do the same. (TODO)
1279 if (MI->getOpcode() != TargetOpcode::STATEPOINT) {
1280 auto Flags = MachineMemOperand::MOLoad;
1282 MachinePointerInfo::getFixedStack(MF, FI), Flags,
1284 MIB->addMemOperand(MF, MMO);
1285 }
1286 }
1288 MI->eraseFromParent();
1289 return MBB;
1290}
1291
1292/// findRepresentativeClass - Return the largest legal super-reg register class
1293/// of the register class for the specified type and its associated "cost".
1294// This function is in TargetLowering because it uses RegClassForVT which would
1295// need to be moved to TargetRegisterInfo and would necessitate moving
1296// isTypeLegal over as well - a massive change that would just require
1297// TargetLowering having a TargetRegisterInfo class member that it would use.
1298std::pair<const TargetRegisterClass *, uint8_t>
1300 MVT VT) const {
1301 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
1302 if (!RC)
1303 return std::make_pair(RC, 0);
1304
1305 // Compute the set of all super-register classes.
1306 BitVector SuperRegRC(TRI->getNumRegClasses());
1307 for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI)
1308 SuperRegRC.setBitsInMask(RCI.getMask());
1309
1310 // Find the first legal register class with the largest spill size.
1311 const TargetRegisterClass *BestRC = RC;
1312 for (unsigned i : SuperRegRC.set_bits()) {
1313 const TargetRegisterClass *SuperRC = TRI->getRegClass(i);
1314 // We want the largest possible spill size.
1315 if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC))
1316 continue;
1317 if (!isLegalRC(*TRI, *SuperRC))
1318 continue;
1319 BestRC = SuperRC;
1320 }
1321 return std::make_pair(BestRC, 1);
1322}
1323
1324/// computeRegisterProperties - Once all of the register classes are added,
1325/// this allows us to compute derived properties we expose.
1327 const TargetRegisterInfo *TRI) {
1329 "Too many value types for ValueTypeActions to hold!");
1330
1331 // Everything defaults to needing one register.
1332 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
1333 NumRegistersForVT[i] = 1;
1334 RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;
1335 }
1336 // ...except isVoid, which doesn't need any registers.
1337 NumRegistersForVT[MVT::isVoid] = 0;
1338
1339 // Find the largest integer register class.
1340 unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;
1341 for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg)
1342 assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
1343
1344 // Every integer value type larger than this largest register takes twice as
1345 // many registers to represent as the previous ValueType.
1346 for (unsigned ExpandedReg = LargestIntReg + 1;
1347 ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) {
1348 NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
1349 RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;
1350 TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);
1351 ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,
1353 }
1354
1355 // Inspect all of the ValueType's smaller than the largest integer
1356 // register to see which ones need promotion.
1357 unsigned LegalIntReg = LargestIntReg;
1358 for (unsigned IntReg = LargestIntReg - 1;
1359 IntReg >= (unsigned)MVT::i1; --IntReg) {
1360 MVT IVT = (MVT::SimpleValueType)IntReg;
1361 if (isTypeLegal(IVT)) {
1362 LegalIntReg = IntReg;
1363 } else {
1364 RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
1365 (MVT::SimpleValueType)LegalIntReg;
1366 ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
1367 }
1368 }
1369
1370 // ppcf128 type is really two f64's.
1371 if (!isTypeLegal(MVT::ppcf128)) {
1372 if (isTypeLegal(MVT::f64)) {
1373 NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];
1374 RegisterTypeForVT[MVT::ppcf128] = MVT::f64;
1375 TransformToType[MVT::ppcf128] = MVT::f64;
1376 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);
1377 } else {
1378 NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128];
1379 RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128];
1380 TransformToType[MVT::ppcf128] = MVT::i128;
1381 ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat);
1382 }
1383 }
1384
1385 // Decide how to handle f128. If the target does not have native f128 support,
1386 // expand it to i128 and we will be generating soft float library calls.
1387 if (!isTypeLegal(MVT::f128)) {
1388 NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];
1389 RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];
1390 TransformToType[MVT::f128] = MVT::i128;
1391 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
1392 }
1393
1394 // Decide how to handle f80. If the target does not have native f80 support,
1395 // expand it to i96 and we will be generating soft float library calls.
1396 if (!isTypeLegal(MVT::f80)) {
1397 NumRegistersForVT[MVT::f80] = 3*NumRegistersForVT[MVT::i32];
1398 RegisterTypeForVT[MVT::f80] = RegisterTypeForVT[MVT::i32];
1399 TransformToType[MVT::f80] = MVT::i32;
1400 ValueTypeActions.setTypeAction(MVT::f80, TypeSoftenFloat);
1401 }
1402
1403 // Decide how to handle f64. If the target does not have native f64 support,
1404 // expand it to i64 and we will be generating soft float library calls.
1405 if (!isTypeLegal(MVT::f64)) {
1406 NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
1407 RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
1408 TransformToType[MVT::f64] = MVT::i64;
1409 ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);
1410 }
1411
1412 // Decide how to handle f32. If the target does not have native f32 support,
1413 // expand it to i32 and we will be generating soft float library calls.
1414 if (!isTypeLegal(MVT::f32)) {
1415 NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
1416 RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
1417 TransformToType[MVT::f32] = MVT::i32;
1418 ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);
1419 }
1420
1421 // Decide how to handle f16. If the target does not have native f16 support,
1422 // promote it to f32, because there are no f16 library calls (except for
1423 // conversions).
1424 if (!isTypeLegal(MVT::f16)) {
1425 // Allow targets to control how we legalize half.
1426 if (softPromoteHalfType()) {
1427 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16];
1428 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16];
1429 TransformToType[MVT::f16] = MVT::f32;
1430 ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf);
1431 } else {
1432 NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];
1433 RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];
1434 TransformToType[MVT::f16] = MVT::f32;
1435 ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);
1436 }
1437 }
1438
1439 // Decide how to handle bf16. If the target does not have native bf16 support,
1440 // promote it to f32, because there are no bf16 library calls (except for
1441 // converting from f32 to bf16).
1442 if (!isTypeLegal(MVT::bf16)) {
1443 NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];
1444 RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];
1445 TransformToType[MVT::bf16] = MVT::f32;
1446 ValueTypeActions.setTypeAction(MVT::bf16, TypeSoftPromoteHalf);
1447 }
1448
1449 // Loop over all of the vector value types to see which need transformations.
1450 for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
1451 i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
1452 MVT VT = (MVT::SimpleValueType) i;
1453 if (isTypeLegal(VT))
1454 continue;
1455
1456 MVT EltVT = VT.getVectorElementType();
1458 bool IsLegalWiderType = false;
1459 bool IsScalable = VT.isScalableVector();
1460 LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT);
1461 switch (PreferredAction) {
1462 case TypePromoteInteger: {
1463 MVT::SimpleValueType EndVT = IsScalable ?
1464 MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE :
1465 MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE;
1466 // Try to promote the elements of integer vectors. If no legal
1467 // promotion was found, fall through to the widen-vector method.
1468 for (unsigned nVT = i + 1;
1469 (MVT::SimpleValueType)nVT <= EndVT; ++nVT) {
1470 MVT SVT = (MVT::SimpleValueType) nVT;
1471 // Promote vectors of integers to vectors with the same number
1472 // of elements, with a wider element type.
1473 if (SVT.getScalarSizeInBits() > EltVT.getFixedSizeInBits() &&
1474 SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) {
1475 TransformToType[i] = SVT;
1476 RegisterTypeForVT[i] = SVT;
1477 NumRegistersForVT[i] = 1;
1478 ValueTypeActions.setTypeAction(VT, TypePromoteInteger);
1479 IsLegalWiderType = true;
1480 break;
1481 }
1482 }
1483 if (IsLegalWiderType)
1484 break;
1485 [[fallthrough]];
1486 }
1487
1488 case TypeWidenVector:
1489 if (isPowerOf2_32(EC.getKnownMinValue())) {
1490 // Try to widen the vector.
1491 for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
1492 MVT SVT = (MVT::SimpleValueType) nVT;
1493 if (SVT.getVectorElementType() == EltVT &&
1494 SVT.isScalableVector() == IsScalable &&
1496 EC.getKnownMinValue() &&
1497 isTypeLegal(SVT)) {
1498 TransformToType[i] = SVT;
1499 RegisterTypeForVT[i] = SVT;
1500 NumRegistersForVT[i] = 1;
1501 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1502 IsLegalWiderType = true;
1503 break;
1504 }
1505 }
1506 if (IsLegalWiderType)
1507 break;
1508 } else {
1509 // Only widen to the next power of 2 to keep consistency with EVT.
1510 MVT NVT = VT.getPow2VectorType();
1511 if (isTypeLegal(NVT)) {
1512 TransformToType[i] = NVT;
1513 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1514 RegisterTypeForVT[i] = NVT;
1515 NumRegistersForVT[i] = 1;
1516 break;
1517 }
1518 }
1519 [[fallthrough]];
1520
1521 case TypeSplitVector:
1522 case TypeScalarizeVector: {
1523 MVT IntermediateVT;
1524 MVT RegisterVT;
1525 unsigned NumIntermediates;
1526 unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT,
1527 NumIntermediates, RegisterVT, this);
1528 NumRegistersForVT[i] = NumRegisters;
1529 assert(NumRegistersForVT[i] == NumRegisters &&
1530 "NumRegistersForVT size cannot represent NumRegisters!");
1531 RegisterTypeForVT[i] = RegisterVT;
1532
1533 MVT NVT = VT.getPow2VectorType();
1534 if (NVT == VT) {
1535 // Type is already a power of 2. The default action is to split.
1536 TransformToType[i] = MVT::Other;
1537 if (PreferredAction == TypeScalarizeVector)
1538 ValueTypeActions.setTypeAction(VT, TypeScalarizeVector);
1539 else if (PreferredAction == TypeSplitVector)
1540 ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1541 else if (EC.getKnownMinValue() > 1)
1542 ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1543 else
1544 ValueTypeActions.setTypeAction(VT, EC.isScalable()
1547 } else {
1548 TransformToType[i] = NVT;
1549 ValueTypeActions.setTypeAction(VT, TypeWidenVector);
1550 }
1551 break;
1552 }
1553 default:
1554 llvm_unreachable("Unknown vector legalization action!");
1555 }
1556 }
1557
1558 // Determine the 'representative' register class for each value type.
1559 // An representative register class is the largest (meaning one which is
1560 // not a sub-register class / subreg register class) legal register class for
1561 // a group of value types. For example, on i386, i8, i16, and i32
1562 // representative would be GR32; while on x86_64 it's GR64.
1563 for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
1564 const TargetRegisterClass* RRC;
1565 uint8_t Cost;
1567 RepRegClassForVT[i] = RRC;
1568 RepRegClassCostForVT[i] = Cost;
1569 }
1570}
1571
1573 EVT VT) const {
1574 assert(!VT.isVector() && "No default SetCC type for vectors!");
1575 return getPointerTy(DL).SimpleTy;
1576}
1577
1579 return MVT::i32; // return the default value
1580}
1581
1582/// getVectorTypeBreakdown - Vector types are broken down into some number of
1583/// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
1584/// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
1585/// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
1586///
1587/// This method returns the number of registers needed, and the VT for each
1588/// register. It also returns the VT and quantity of the intermediate values
1589/// before they are promoted/expanded.
1591 EVT VT, EVT &IntermediateVT,
1592 unsigned &NumIntermediates,
1593 MVT &RegisterVT) const {
1594 ElementCount EltCnt = VT.getVectorElementCount();
1595
1596 // If there is a wider vector type with the same element type as this one,
1597 // or a promoted vector type that has the same number of elements which
1598 // are wider, then we should convert to that legal vector type.
1599 // This handles things like <2 x float> -> <4 x float> and
1600 // <4 x i1> -> <4 x i32>.
1602 if (!EltCnt.isScalar() &&
1603 (TA == TypeWidenVector || TA == TypePromoteInteger)) {
1604 EVT RegisterEVT = getTypeToTransformTo(Context, VT);
1605 if (isTypeLegal(RegisterEVT)) {
1606 IntermediateVT = RegisterEVT;
1607 RegisterVT = RegisterEVT.getSimpleVT();
1608 NumIntermediates = 1;
1609 return 1;
1610 }
1611 }
1612
1613 // Figure out the right, legal destination reg to copy into.
1614 EVT EltTy = VT.getVectorElementType();
1615
1616 unsigned NumVectorRegs = 1;
1617
1618 // Scalable vectors cannot be scalarized, so handle the legalisation of the
1619 // types like done elsewhere in SelectionDAG.
1620 if (EltCnt.isScalable()) {
1621 LegalizeKind LK;
1622 EVT PartVT = VT;
1623 do {
1624 // Iterate until we've found a legal (part) type to hold VT.
1625 LK = getTypeConversion(Context, PartVT);
1626 PartVT = LK.second;
1627 } while (LK.first != TypeLegal);
1628
1629 if (!PartVT.isVector()) {
1631 "Don't know how to legalize this scalable vector type");
1632 }
1633
1634 NumIntermediates =
1637 IntermediateVT = PartVT;
1638 RegisterVT = getRegisterType(Context, IntermediateVT);
1639 return NumIntermediates;
1640 }
1641
1642 // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally
1643 // we could break down into LHS/RHS like LegalizeDAG does.
1644 if (!isPowerOf2_32(EltCnt.getKnownMinValue())) {
1645 NumVectorRegs = EltCnt.getKnownMinValue();
1646 EltCnt = ElementCount::getFixed(1);
1647 }
1648
1649 // Divide the input until we get to a supported size. This will always
1650 // end with a scalar if the target doesn't support vectors.
1651 while (EltCnt.getKnownMinValue() > 1 &&
1652 !isTypeLegal(EVT::getVectorVT(Context, EltTy, EltCnt))) {
1653 EltCnt = EltCnt.divideCoefficientBy(2);
1654 NumVectorRegs <<= 1;
1655 }
1656
1657 NumIntermediates = NumVectorRegs;
1658
1659 EVT NewVT = EVT::getVectorVT(Context, EltTy, EltCnt);
1660 if (!isTypeLegal(NewVT))
1661 NewVT = EltTy;
1662 IntermediateVT = NewVT;
1663
1664 MVT DestVT = getRegisterType(Context, NewVT);
1665 RegisterVT = DestVT;
1666
1667 if (EVT(DestVT).bitsLT(NewVT)) { // Value is expanded, e.g. i64 -> i16.
1668 TypeSize NewVTSize = NewVT.getSizeInBits();
1669 // Convert sizes such as i33 to i64.
1670 if (!llvm::has_single_bit<uint32_t>(NewVTSize.getKnownMinValue()))
1671 NewVTSize = NewVTSize.coefficientNextPowerOf2();
1672 return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
1673 }
1674
1675 // Otherwise, promotion or legal types use the same number of registers as
1676 // the vector decimated to the appropriate level.
1677 return NumVectorRegs;
1678}
1679
1681 uint64_t NumCases,
1682 uint64_t Range,
1683 ProfileSummaryInfo *PSI,
1684 BlockFrequencyInfo *BFI) const {
1685 // FIXME: This function check the maximum table size and density, but the
1686 // minimum size is not checked. It would be nice if the minimum size is
1687 // also combined within this function. Currently, the minimum size check is
1688 // performed in findJumpTable() in SelectionDAGBuiler and
1689 // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
1690 const bool OptForSize =
1691 SI->getParent()->getParent()->hasOptSize() ||
1692 llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI);
1693 const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);
1694 const unsigned MaxJumpTableSize = getMaximumJumpTableSize();
1695
1696 // Check whether the number of cases is small enough and
1697 // the range is dense enough for a jump table.
1698 return (OptForSize || Range <= MaxJumpTableSize) &&
1699 (NumCases * 100 >= Range * MinDensity);
1700}
1701
1703 EVT ConditionVT) const {
1704 return getRegisterType(Context, ConditionVT);
1705}
1706
1707/// Get the EVTs and ArgFlags collections that represent the legalized return
1708/// type of the given function. This does not require a DAG or a return value,
1709/// and is suitable for use before any DAGs for the function are constructed.
1710/// TODO: Move this out of TargetLowering.cpp.
1712 AttributeList attr,
1714 const TargetLowering &TLI, const DataLayout &DL) {
1715 SmallVector<EVT, 4> ValueVTs;
1716 ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
1717 unsigned NumValues = ValueVTs.size();
1718 if (NumValues == 0) return;
1719
1720 for (unsigned j = 0, f = NumValues; j != f; ++j) {
1721 EVT VT = ValueVTs[j];
1722 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1723
1724 if (attr.hasRetAttr(Attribute::SExt))
1725 ExtendKind = ISD::SIGN_EXTEND;
1726 else if (attr.hasRetAttr(Attribute::ZExt))
1727 ExtendKind = ISD::ZERO_EXTEND;
1728
1729 // FIXME: C calling convention requires the return type to be promoted to
1730 // at least 32-bit. But this is not necessary for non-C calling
1731 // conventions. The frontend should mark functions whose return values
1732 // require promoting with signext or zeroext attributes.
1733 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) {
1734 MVT MinVT = TLI.getRegisterType(MVT::i32);
1735 if (VT.bitsLT(MinVT))
1736 VT = MinVT;
1737 }
1738
1739 unsigned NumParts =
1740 TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT);
1741 MVT PartVT =
1742 TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT);
1743
1744 // 'inreg' on function refers to return value
1746 if (attr.hasRetAttr(Attribute::InReg))
1747 Flags.setInReg();
1748
1749 // Propagate extension type if any
1750 if (attr.hasRetAttr(Attribute::SExt))
1751 Flags.setSExt();
1752 else if (attr.hasRetAttr(Attribute::ZExt))
1753 Flags.setZExt();
1754
1755 for (unsigned i = 0; i < NumParts; ++i)
1756 Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0));
1757 }
1758}
1759
1760/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1761/// function arguments in the caller parameter area. This is the actual
1762/// alignment, not its logarithm.
1764 const DataLayout &DL) const {
1765 return DL.getABITypeAlign(Ty).value();
1766}
1767
1769 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
1770 Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
1771 // Check if the specified alignment is sufficient based on the data layout.
1772 // TODO: While using the data layout works in practice, a better solution
1773 // would be to implement this check directly (make this a virtual function).
1774 // For example, the ABI alignment may change based on software platform while
1775 // this function should only be affected by hardware implementation.
1776 Type *Ty = VT.getTypeForEVT(Context);
1777 if (VT.isZeroSized() || Alignment >= DL.getABITypeAlign(Ty)) {
1778 // Assume that an access that meets the ABI-specified alignment is fast.
1779 if (Fast != nullptr)
1780 *Fast = 1;
1781 return true;
1782 }
1783
1784 // This is a misaligned access.
1785 return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
1786}
1787
1789 LLVMContext &Context, const DataLayout &DL, EVT VT,
1790 const MachineMemOperand &MMO, unsigned *Fast) const {
1792 MMO.getAlign(), MMO.getFlags(), Fast);
1793}
1794
1796 const DataLayout &DL, EVT VT,
1797 unsigned AddrSpace, Align Alignment,
1799 unsigned *Fast) const {
1800 return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment,
1801 Flags, Fast);
1802}
1803
1805 const DataLayout &DL, EVT VT,
1806 const MachineMemOperand &MMO,
1807 unsigned *Fast) const {
1808 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
1809 MMO.getFlags(), Fast);
1810}
1811
1813 const DataLayout &DL, LLT Ty,
1814 const MachineMemOperand &MMO,
1815 unsigned *Fast) const {
1817 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
1818 MMO.getFlags(), Fast);
1819}
1820
1821//===----------------------------------------------------------------------===//
1822// TargetTransformInfo Helpers
1823//===----------------------------------------------------------------------===//
1824
1826 enum InstructionOpcodes {
1827#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
1828#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
1829#include "llvm/IR/Instruction.def"
1830 };
1831 switch (static_cast<InstructionOpcodes>(Opcode)) {
1832 case Ret: return 0;
1833 case Br: return 0;
1834 case Switch: return 0;
1835 case IndirectBr: return 0;
1836 case Invoke: return 0;
1837 case CallBr: return 0;
1838 case Resume: return 0;
1839 case Unreachable: return 0;
1840 case CleanupRet: return 0;
1841 case CatchRet: return 0;
1842 case CatchPad: return 0;
1843 case CatchSwitch: return 0;
1844 case CleanupPad: return 0;
1845 case FNeg: return ISD::FNEG;
1846 case Add: return ISD::ADD;
1847 case FAdd: return ISD::FADD;
1848 case Sub: return ISD::SUB;
1849 case FSub: return ISD::FSUB;
1850 case Mul: return ISD::MUL;
1851 case FMul: return ISD::FMUL;
1852 case UDiv: return ISD::UDIV;
1853 case SDiv: return ISD::SDIV;
1854 case FDiv: return ISD::FDIV;
1855 case URem: return ISD::UREM;
1856 case SRem: return ISD::SREM;
1857 case FRem: return ISD::FREM;
1858 case Shl: return ISD::SHL;
1859 case LShr: return ISD::SRL;
1860 case AShr: return ISD::SRA;
1861 case And: return ISD::AND;
1862 case Or: return ISD::OR;
1863 case Xor: return ISD::XOR;
1864 case Alloca: return 0;
1865 case Load: return ISD::LOAD;
1866 case Store: return ISD::STORE;
1867 case GetElementPtr: return 0;
1868 case Fence: return 0;
1869 case AtomicCmpXchg: return 0;
1870 case AtomicRMW: return 0;
1871 case Trunc: return ISD::TRUNCATE;
1872 case ZExt: return ISD::ZERO_EXTEND;
1873 case SExt: return ISD::SIGN_EXTEND;
1874 case FPToUI: return ISD::FP_TO_UINT;
1875 case FPToSI: return ISD::FP_TO_SINT;
1876 case UIToFP: return ISD::UINT_TO_FP;
1877 case SIToFP: return ISD::SINT_TO_FP;
1878 case FPTrunc: return ISD::FP_ROUND;
1879 case FPExt: return ISD::FP_EXTEND;
1880 case PtrToInt: return ISD::BITCAST;
1881 case IntToPtr: return ISD::BITCAST;
1882 case BitCast: return ISD::BITCAST;
1883 case AddrSpaceCast: return ISD::ADDRSPACECAST;
1884 case ICmp: return ISD::SETCC;
1885 case FCmp: return ISD::SETCC;
1886 case PHI: return 0;
1887 case Call: return 0;
1888 case Select: return ISD::SELECT;
1889 case UserOp1: return 0;
1890 case UserOp2: return 0;
1891 case VAArg: return 0;
1892 case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
1893 case InsertElement: return ISD::INSERT_VECTOR_ELT;
1894 case ShuffleVector: return ISD::VECTOR_SHUFFLE;
1895 case ExtractValue: return ISD::MERGE_VALUES;
1896 case InsertValue: return ISD::MERGE_VALUES;
1897 case LandingPad: return 0;
1898 case Freeze: return ISD::FREEZE;
1899 }
1900
1901 llvm_unreachable("Unknown instruction type encountered!");
1902}
1903
1904Value *
1906 bool UseTLS) const {
1907 // compiler-rt provides a variable with a magic name. Targets that do not
1908 // link with compiler-rt may also provide such a variable.
1909 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1910 const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";
1911 auto UnsafeStackPtr =
1912 dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar));
1913
1914 Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
1915
1916 if (!UnsafeStackPtr) {
1917 auto TLSModel = UseTLS ?
1920 // The global variable is not defined yet, define it ourselves.
1921 // We use the initial-exec TLS model because we do not support the
1922 // variable living anywhere other than in the main executable.
1923 UnsafeStackPtr = new GlobalVariable(
1924 *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr,
1925 UnsafeStackPtrVar, nullptr, TLSModel);
1926 } else {
1927 // The variable exists, check its type and attributes.
1928 if (UnsafeStackPtr->getValueType() != StackPtrTy)
1929 report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type");
1930 if (UseTLS != UnsafeStackPtr->isThreadLocal())
1931 report_fatal_error(Twine(UnsafeStackPtrVar) + " must " +
1932 (UseTLS ? "" : "not ") + "be thread-local");
1933 }
1934 return UnsafeStackPtr;
1935}
1936
1937Value *
1939 if (!TM.getTargetTriple().isAndroid())
1940 return getDefaultSafeStackPointerLocation(IRB, true);
1941
1942 // Android provides a libc function to retrieve the address of the current
1943 // thread's unsafe stack pointer.
1944 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1945 Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
1946 FunctionCallee Fn = M->getOrInsertFunction("__safestack_pointer_address",
1947 StackPtrTy->getPointerTo(0));
1948 return IRB.CreateCall(Fn);
1949}
1950
1951//===----------------------------------------------------------------------===//
1952// Loop Strength Reduction hooks
1953//===----------------------------------------------------------------------===//
1954
1955/// isLegalAddressingMode - Return true if the addressing mode represented
1956/// by AM is legal for this target, for a load/store of the specified type.
1958 const AddrMode &AM, Type *Ty,
1959 unsigned AS, Instruction *I) const {
1960 // The default implementation of this implements a conservative RISCy, r+r and
1961 // r+i addr mode.
1962
1963 // Allows a sign-extended 16-bit immediate field.
1964 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
1965 return false;
1966
1967 // No global is ever allowed as a base.
1968 if (AM.BaseGV)
1969 return false;
1970
1971 // Only support r+r,
1972 switch (AM.Scale) {
1973 case 0: // "r+i" or just "i", depending on HasBaseReg.
1974 break;
1975 case 1:
1976 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
1977 return false;
1978 // Otherwise we have r+r or r+i.
1979 break;
1980 case 2:
1981 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
1982 return false;
1983 // Allow 2*r as r+r.
1984 break;
1985 default: // Don't allow n * r
1986 return false;
1987 }
1988
1989 return true;
1990}
1991
1992//===----------------------------------------------------------------------===//
1993// Stack Protector
1994//===----------------------------------------------------------------------===//
1995
1996// For OpenBSD return its special guard variable. Otherwise return nullptr,
1997// so that SelectionDAG handle SSP.
1999 if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
2000 Module &M = *IRB.GetInsertBlock()->getParent()->getParent();
2001 PointerType *PtrTy = Type::getInt8PtrTy(M.getContext());
2002 Constant *C = M.getOrInsertGlobal("__guard_local", PtrTy);
2003 if (GlobalVariable *G = dyn_cast_or_null<GlobalVariable>(C))
2004 G->setVisibility(GlobalValue::HiddenVisibility);
2005 return C;
2006 }
2007 return nullptr;
2008}
2009
2010// Currently only support "standard" __stack_chk_guard.
2011// TODO: add LOAD_STACK_GUARD support.
2013 if (!M.getNamedValue("__stack_chk_guard")) {
2014 auto *GV = new GlobalVariable(M, Type::getInt8PtrTy(M.getContext()), false,
2016 "__stack_chk_guard");
2017
2018 // FreeBSD has "__stack_chk_guard" defined externally on libc.so
2019 if (M.getDirectAccessExternalData() &&
2021 !TM.getTargetTriple().isOSFreeBSD() &&
2023 GV->setDSOLocal(true);
2024 }
2025}
2026
2027// Currently only support "standard" __stack_chk_guard.
2028// TODO: add LOAD_STACK_GUARD support.
2030 return M.getNamedValue("__stack_chk_guard");
2031}
2032
2034 return nullptr;
2035}
2036
2039}
2040
2043}
2044
2045unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const {
2046 return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity;
2047}
2048
2050 return MaximumJumpTableSize;
2051}
2052
2055}
2056
2059}
2060
2062 if (TM.Options.LoopAlignment)
2063 return Align(TM.Options.LoopAlignment);
2064 return PrefLoopAlignment;
2065}
2066
2068 MachineBasicBlock *MBB) const {
2069 return MaxBytesForAlignment;
2070}
2071
2072//===----------------------------------------------------------------------===//
2073// Reciprocal Estimates
2074//===----------------------------------------------------------------------===//
2075
2076/// Get the reciprocal estimate attribute string for a function that will
2077/// override the target defaults.
2079 const Function &F = MF.getFunction();
2080 return F.getFnAttribute("reciprocal-estimates").getValueAsString();
2081}
2082
2083/// Construct a string for the given reciprocal operation of the given type.
2084/// This string should match the corresponding option to the front-end's
2085/// "-mrecip" flag assuming those strings have been passed through in an
2086/// attribute string. For example, "vec-divf" for a division of a vXf32.
2087static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
2088 std::string Name = VT.isVector() ? "vec-" : "";
2089
2090 Name += IsSqrt ? "sqrt" : "div";
2091
2092 // TODO: Handle other float types?
2093 if (VT.getScalarType() == MVT::f64) {
2094 Name += "d";
2095 } else if (VT.getScalarType() == MVT::f16) {
2096 Name += "h";
2097 } else {
2098 assert(VT.getScalarType() == MVT::f32 &&
2099 "Unexpected FP type for reciprocal estimate");
2100 Name += "f";
2101 }
2102
2103 return Name;
2104}
2105
2106/// Return the character position and value (a single numeric character) of a
2107/// customized refinement operation in the input string if it exists. Return
2108/// false if there is no customized refinement step count.
2109static bool parseRefinementStep(StringRef In, size_t &Position,
2110 uint8_t &Value) {
2111 const char RefStepToken = ':';
2112 Position = In.find(RefStepToken);
2113 if (Position == StringRef::npos)
2114 return false;
2115
2116 StringRef RefStepString = In.substr(Position + 1);
2117 // Allow exactly one numeric character for the additional refinement
2118 // step parameter.
2119 if (RefStepString.size() == 1) {
2120 char RefStepChar = RefStepString[0];
2121 if (isDigit(RefStepChar)) {
2122 Value = RefStepChar - '0';
2123 return true;
2124 }
2125 }
2126 report_fatal_error("Invalid refinement step for -recip.");
2127}
2128
2129/// For the input attribute string, return one of the ReciprocalEstimate enum
2130/// status values (enabled, disabled, or not specified) for this operation on
2131/// the specified data type.
2132static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
2133 if (Override.empty())
2135
2136 SmallVector<StringRef, 4> OverrideVector;
2137 Override.split(OverrideVector, ',');
2138 unsigned NumArgs = OverrideVector.size();
2139
2140 // Check if "all", "none", or "default" was specified.
2141 if (NumArgs == 1) {
2142 // Look for an optional setting of the number of refinement steps needed
2143 // for this type of reciprocal operation.
2144 size_t RefPos;
2145 uint8_t RefSteps;
2146 if (parseRefinementStep(Override, RefPos, RefSteps)) {
2147 // Split the string for further processing.
2148 Override = Override.substr(0, RefPos);
2149 }
2150
2151 // All reciprocal types are enabled.
2152 if (Override == "all")
2154
2155 // All reciprocal types are disabled.
2156 if (Override == "none")
2158
2159 // Target defaults for enablement are used.
2160 if (Override == "default")
2162 }
2163
2164 // The attribute string may omit the size suffix ('f'/'d').
2165 std::string VTName = getReciprocalOpName(IsSqrt, VT);
2166 std::string VTNameNoSize = VTName;
2167 VTNameNoSize.pop_back();
2168 static const char DisabledPrefix = '!';
2169
2170 for (StringRef RecipType : OverrideVector) {
2171 size_t RefPos;
2172 uint8_t RefSteps;
2173 if (parseRefinementStep(RecipType, RefPos, RefSteps))
2174 RecipType = RecipType.substr(0, RefPos);
2175
2176 // Ignore the disablement token for string matching.
2177 bool IsDisabled = RecipType[0] == DisabledPrefix;
2178 if (IsDisabled)
2179 RecipType = RecipType.substr(1);
2180
2181 if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
2184 }
2185
2187}
2188
2189/// For the input attribute string, return the customized refinement step count
2190/// for this operation on the specified data type. If the step count does not
2191/// exist, return the ReciprocalEstimate enum value for unspecified.
2192static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
2193 if (Override.empty())
2195
2196 SmallVector<StringRef, 4> OverrideVector;
2197 Override.split(OverrideVector, ',');
2198 unsigned NumArgs = OverrideVector.size();
2199
2200 // Check if "all", "default", or "none" was specified.
2201 if (NumArgs == 1) {
2202 // Look for an optional setting of the number of refinement steps needed
2203 // for this type of reciprocal operation.
2204 size_t RefPos;
2205 uint8_t RefSteps;
2206 if (!parseRefinementStep(Override, RefPos, RefSteps))
2208
2209 // Split the string for further processing.
2210 Override = Override.substr(0, RefPos);
2211 assert(Override != "none" &&
2212 "Disabled reciprocals, but specifed refinement steps?");
2213
2214 // If this is a general override, return the specified number of steps.
2215 if (Override == "all" || Override == "default")
2216 return RefSteps;
2217 }
2218
2219 // The attribute string may omit the size suffix ('f'/'d').
2220 std::string VTName = getReciprocalOpName(IsSqrt, VT);
2221 std::string VTNameNoSize = VTName;
2222 VTNameNoSize.pop_back();
2223
2224 for (StringRef RecipType : OverrideVector) {
2225 size_t RefPos;
2226 uint8_t RefSteps;
2227 if (!parseRefinementStep(RecipType, RefPos, RefSteps))
2228 continue;
2229
2230 RecipType = RecipType.substr(0, RefPos);
2231 if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
2232 return RefSteps;
2233 }
2234
2236}
2237
2239 MachineFunction &MF) const {
2240 return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
2241}
2242
2244 MachineFunction &MF) const {
2245 return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
2246}
2247
2249 MachineFunction &MF) const {
2250 return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
2251}
2252
2254 MachineFunction &MF) const {
2255 return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
2256}
2257
2259 EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG,
2260 const MachineMemOperand &MMO) const {
2261 // Single-element vectors are scalarized, so we should generally avoid having
2262 // any memory operations on such types, as they would get scalarized too.
2263 if (LoadVT.isFixedLengthVector() && BitcastVT.isFixedLengthVector() &&
2264 BitcastVT.getVectorNumElements() == 1)
2265 return false;
2266
2267 // Don't do if we could do an indexed load on the original type, but not on
2268 // the new one.
2269 if (!LoadVT.isSimple() || !BitcastVT.isSimple())
2270 return true;
2271
2272 MVT LoadMVT = LoadVT.getSimpleVT();
2273
2274 // Don't bother doing this if it's just going to be promoted again later, as
2275 // doing so might interfere with other combines.
2276 if (getOperationAction(ISD::LOAD, LoadMVT) == Promote &&
2277 getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())
2278 return false;
2279
2280 unsigned Fast = 0;
2281 return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,
2282 MMO, &Fast) &&
2283 Fast;
2284}
2285
2288}
2289
2291 const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC,
2292 const TargetLibraryInfo *LibInfo) const {
2294 if (LI.isVolatile())
2296
2297 if (LI.hasMetadata(LLVMContext::MD_nontemporal))
2299
2300 if (LI.hasMetadata(LLVMContext::MD_invariant_load))
2302
2304 LI.getAlign(), DL, &LI, AC,
2305 /*DT=*/nullptr, LibInfo))
2307
2308 Flags |= getTargetMMOFlags(LI);
2309 return Flags;
2310}
2311
2314 const DataLayout &DL) const {
2316
2317 if (SI.isVolatile())
2319
2320 if (SI.hasMetadata(LLVMContext::MD_nontemporal))
2322
2323 // FIXME: Not preserving dereferenceable
2324 Flags |= getTargetMMOFlags(SI);
2325 return Flags;
2326}
2327
2330 const DataLayout &DL) const {
2332
2333 if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) {
2334 if (RMW->isVolatile())
2336 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) {
2337 if (CmpX->isVolatile())
2339 } else
2340 llvm_unreachable("not an atomic instruction");
2341
2342 // FIXME: Not preserving dereferenceable
2343 Flags |= getTargetMMOFlags(AI);
2344 return Flags;
2345}
2346
2348 Instruction *Inst,
2349 AtomicOrdering Ord) const {
2350 if (isReleaseOrStronger(Ord) && Inst->hasAtomicStore())
2351 return Builder.CreateFence(Ord);
2352 else
2353 return nullptr;
2354}
2355
2357 Instruction *Inst,
2358 AtomicOrdering Ord) const {
2359 if (isAcquireOrStronger(Ord))
2360 return Builder.CreateFence(Ord);
2361 else
2362 return nullptr;
2363}
2364
2365//===----------------------------------------------------------------------===//
2366// GlobalISel Hooks
2367//===----------------------------------------------------------------------===//
2368
2370 const TargetTransformInfo *TTI) const {
2371 auto &MF = *MI.getMF();
2372 auto &MRI = MF.getRegInfo();
2373 // Assuming a spill and reload of a value has a cost of 1 instruction each,
2374 // this helper function computes the maximum number of uses we should consider
2375 // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We
2376 // break even in terms of code size when the original MI has 2 users vs
2377 // choosing to potentially spill. Any more than 2 users we we have a net code
2378 // size increase. This doesn't take into account register pressure though.
2379 auto maxUses = [](unsigned RematCost) {
2380 // A cost of 1 means remats are basically free.
2381 if (RematCost == 1)
2382 return std::numeric_limits<unsigned>::max();
2383 if (RematCost == 2)
2384 return 2U;
2385
2386 // Remat is too expensive, only sink if there's one user.
2387 if (RematCost > 2)
2388 return 1U;
2389 llvm_unreachable("Unexpected remat cost");
2390 };
2391
2392 switch (MI.getOpcode()) {
2393 default:
2394 return false;
2395 // Constants-like instructions should be close to their users.
2396 // We don't want long live-ranges for them.
2397 case TargetOpcode::G_CONSTANT:
2398 case TargetOpcode::G_FCONSTANT:
2399 case TargetOpcode::G_FRAME_INDEX:
2400 case TargetOpcode::G_INTTOPTR:
2401 return true;
2402 case TargetOpcode::G_GLOBAL_VALUE: {
2403 unsigned RematCost = TTI->getGISelRematGlobalCost();
2404 Register Reg = MI.getOperand(0).getReg();
2405 unsigned MaxUses = maxUses(RematCost);
2406 if (MaxUses == UINT_MAX)
2407 return true; // Remats are "free" so always localize.
2408 return MRI.hasAtMostUserInstrs(Reg, MaxUses);
2409 }
2410 }
2411}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
Rewrite undef for PHI
assume Assume Builder
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
std::string Name
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
LLVMContext & Context
const char LLVMTargetMachineRef TM
static bool isDigit(const char C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
This file contains some functions that are useful when dealing with strings.
static bool darwinHasSinCos(const Triple &TT)
static cl::opt< bool > JumpIsExpensiveOverride("jump-is-expensive", cl::init(false), cl::desc("Do not create extra branches to split comparison logic."), cl::Hidden)
#define OP_TO_LIBCALL(Name, Enum)
static cl::opt< unsigned > MinimumJumpTableEntries("min-jump-table-entries", cl::init(4), cl::Hidden, cl::desc("Set minimum number of entries to use a jump table."))
static cl::opt< bool > DisableStrictNodeMutation("disable-strictnode-mutation", cl::desc("Don't mutate strict-float node to a legalize node"), cl::init(false), cl::Hidden)
static bool parseRefinementStep(StringRef In, size_t &Position, uint8_t &Value)
Return the character position and value (a single numeric character) of a customized refinement opera...
static cl::opt< unsigned > MaximumJumpTableSize("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden, cl::desc("Set maximum size of jump tables."))
static cl::opt< unsigned > JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden, cl::desc("Minimum density for building a jump table in " "a normal function"))
Minimum jump table density for normal functions.
static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT, TargetLoweringBase *TLI)
static std::string getReciprocalOpName(bool IsSqrt, EVT VT)
Construct a string for the given reciprocal operation of the given type.
#define LCALL5(A)
static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override)
For the input attribute string, return the customized refinement step count for this operation on the...
static void InitCmpLibcallCCs(ISD::CondCode *CCs)
InitCmpLibcallCCs - Set default comparison libcall CC.
static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override)
For the input attribute string, return one of the ReciprocalEstimate enum status values (enabled,...
static StringRef getRecipEstimateForFunc(MachineFunction &MF)
Get the reciprocal estimate attribute string for a function that will override the target defaults.
static cl::opt< unsigned > OptsizeJumpTableDensity("optsize-jump-table-density", cl::init(40), cl::Hidden, cl::desc("Minimum density for building a jump table in " "an optsize function"))
Minimum jump table density for -Os or -Oz functions.
This file describes how to lower LLVM code to machine code.
This pass exposes codegen information to IR-level passes.
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:513
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:718
bool hasRetAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the return value.
Definition: Attributes.h:782
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Definition: BitVector.h:707
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:140
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size in bytes, rounded up to a whole number of bytes.
Definition: DataLayout.cpp:748
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition: TypeSize.h:294
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:291
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:302
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:174
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2374
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:284
@ MAX_INT_BITS
Maximum number of bits that can be specified.
Definition: DerivedTypes.h:52
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:177
Value * getPointerOperand()
Definition: Instructions.h:264
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:214
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:220
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static auto all_valuetypes()
SimpleValueType Iteration.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
MVT getPow2VectorType() const
Widens the length of the given vector MVT up to the nearest power of 2 and returns that type.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool isStatepointSpillSlotObjectIndex(int ObjectIdx) const
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:546
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
A description of a memory reference used in the backend.
unsigned getAddrSpace() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
void freezeReservedRegs(const MachineFunction &)
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Class to represent pointers.
Definition: DerivedTypes.h:643
Analysis providing profile information.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
An instruction for storing to memory.
Definition: Instructions.h:301
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:704
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:575
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
static constexpr size_t npos
Definition: StringRef.h:52
bool isValid() const
Returns true if this iterator is still pointing at a valid entry.
Multiway switch.
Provides information about what library functions are available for the current target.
LegalizeTypeAction getTypeAction(MVT VT) const
void setTypeAction(MVT VT, LegalizeTypeAction Action)
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void initActions()
Initialize all of the actions to default values.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const
Returns the target-specific address of the unsafe stack pointer.
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const
Check whether or not MI needs to be moved close to its uses.
virtual unsigned getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const
Return the maximum amount of bytes allowed to be emitted when padding for alignment.
void setMaximumJumpTableSize(unsigned)
Indicate the maximum number of entries in jump tables.
virtual unsigned getMinimumJumpTableEntries() const
Return lower limit for number of blocks in a jump table.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases, uint64_t Range, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) const
Return true if lowering to a jump table is suitable for a set of case clusters which may contain NumC...
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
virtual bool softPromoteHalfType() const
unsigned getMaximumJumpTableSize() const
Return upper limit for number of entries in a jump table.
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
Value * getDefaultSafeStackPointerLocation(IRBuilderBase &IRB, bool UseTLS) const
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual Align getPrefLoopAlignment(MachineLoop *ML=nullptr) const
Return the preferred loop alignment.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
virtual MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const
Returns preferred type for switch condition.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual bool isJumpTableRelative() const
virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const
Return the type to use for a scalar shift opcode, given the shifted amount type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
unsigned getMinimumJumpTableDensity(bool OptForSize) const
Return lower limit of the density in a jump table.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
TargetLoweringBase(const TargetMachine &TM)
NOTE: The TargetMachine owns TLOF.
LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const
Return pair that represents the legalization kind (first) that needs to happen to EVT (second) in ord...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should continue looking for chain dependencies when trying to find a more...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
virtual Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
virtual Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
Inserts in the IR a target-specific intrinsic specifying a fence.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
std::pair< LegalizeTypeAction, EVT > LegalizeKind
LegalizeKind holds the legalization kind that needs to happen to EVT in order to type-legalize it.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
bool isPositionIndependent() const
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
TargetOptions Options
unsigned LoopAlignment
If greater than 0, override TargetLoweringBase::PrefLoopAlignment.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
unsigned getGISelRematGlobalCost() const
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isWindowsGNUEnvironment() const
Definition: Triple.h:617
bool isAndroid() const
Tests whether the target is Android.
Definition: Triple.h:725
@ aarch64_32
Definition: Triple.h:53
bool isOSFreeBSD() const
Definition: Triple.h:544
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, or DriverKit).
Definition: Triple.h:518
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
constexpr LeafTy coefficientNextPowerOf2() const
Definition: TypeSize.h:242
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:166
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:163
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition: TypeSize.h:234
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
Definition: CallingConv.h:111
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:993
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1328
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1359
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:497
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1258
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1260
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1261
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:269
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:997
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:974
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1026
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1016
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:780
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1243
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:787
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1344
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1348
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1020
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:817
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1358
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1256
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:900
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1257
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:934
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1380
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:662
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1341
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1210
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1345
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:758
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:925
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1259
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:507
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1360
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1353
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1011
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:988
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1255
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1254
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:973
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:777
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1200
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:853
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1237
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1262
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:966
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1204
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:806
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:795
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:885
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:303
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1361
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1252
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:979
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1253
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:833
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1177
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1197
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1002
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:657
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1342
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:279
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1251
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:939
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:866
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:828
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:904
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:852
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1349
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:783
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1329
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1309
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1007
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:313
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1503
static const int LAST_INDEXED_MODE
Definition: ISDOpcodes.h:1454
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< TypeSize > *Offsets, TypeSize StartingOffset)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:122
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx)
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition: MathExtras.h:414
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
Definition: Sequence.h:337
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
Definition: Loads.cpp:199
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
constexpr force_iteration_on_noniterable_enum_t force_iteration_on_noniterable_enum
Definition: Sequence.h:108
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:306
bool isReleaseOrStronger(AtomicOrdering AO)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1741
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ FMul
Product of floats.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
@ FAdd
Sum of floats.
bool isAcquireOrStronger(AtomicOrdering AO)
InstructionCost Cost
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
EVT getPow2VectorType(LLVMContext &Context) const
Widens the length of the given vector EVT up to the nearest power of 2 and returns that type.
Definition: ValueTypes.h:455
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:283
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:333
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition: ValueTypes.h:448
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isFixedLengthVector() const
Definition: ValueTypes.h:170
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
Definition: ValueTypes.h:397
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:306
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:194
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:319
bool isZeroSized() const
Test if the given EVT has zero size, this will fail if called on a scalable type.
Definition: ValueTypes.h:124
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:431
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...