LLVM 19.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/Metadata.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Operator.h"
87#include "llvm/IR/Statepoint.h"
88#include "llvm/IR/Type.h"
89#include "llvm/IR/User.h"
90#include "llvm/IR/Value.h"
91#include "llvm/MC/MCContext.h"
96#include "llvm/Support/Debug.h"
105#include <cstddef>
106#include <iterator>
107#include <limits>
108#include <optional>
109#include <tuple>
110
111using namespace llvm;
112using namespace PatternMatch;
113using namespace SwitchCG;
114
115#define DEBUG_TYPE "isel"
116
117/// LimitFloatPrecision - Generate low-precision inline sequences for
118/// some float libcalls (6, 8 or 12 bits).
119static unsigned LimitFloatPrecision;
120
121static cl::opt<bool>
122 InsertAssertAlign("insert-assert-align", cl::init(true),
123 cl::desc("Insert the experimental `assertalign` node."),
125
127 LimitFPPrecision("limit-float-precision",
128 cl::desc("Generate low-precision inline sequences "
129 "for some float libcalls"),
131 cl::init(0));
132
134 "switch-peel-threshold", cl::Hidden, cl::init(66),
135 cl::desc("Set the case probability threshold for peeling the case from a "
136 "switch statement. A value greater than 100 will void this "
137 "optimization"));
138
139// Limit the width of DAG chains. This is important in general to prevent
140// DAG-based analysis from blowing up. For example, alias analysis and
141// load clustering may not complete in reasonable time. It is difficult to
142// recognize and avoid this situation within each individual analysis, and
143// future analyses are likely to have the same behavior. Limiting DAG width is
144// the safe approach and will be especially important with global DAGs.
145//
146// MaxParallelChains default is arbitrarily high to avoid affecting
147// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
148// sequence over this should have been converted to llvm.memcpy by the
149// frontend. It is easy to induce this behavior with .ll code such as:
150// %buffer = alloca [4096 x i8]
151// %data = load [4096 x i8]* %argPtr
152// store [4096 x i8] %data, [4096 x i8]* %buffer
153static const unsigned MaxParallelChains = 64;
154
156 const SDValue *Parts, unsigned NumParts,
157 MVT PartVT, EVT ValueVT, const Value *V,
158 SDValue InChain,
159 std::optional<CallingConv::ID> CC);
160
161/// getCopyFromParts - Create a value that contains the specified legal parts
162/// combined into the value they represent. If the parts combine to a type
163/// larger than ValueVT then AssertOp can be used to specify whether the extra
164/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
165/// (ISD::AssertSext).
166static SDValue
167getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
168 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
169 SDValue InChain,
170 std::optional<CallingConv::ID> CC = std::nullopt,
171 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
172 // Let the target assemble the parts if it wants to
173 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
174 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
175 PartVT, ValueVT, CC))
176 return Val;
177
178 if (ValueVT.isVector())
179 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
180 InChain, CC);
181
182 assert(NumParts > 0 && "No parts to assemble!");
183 SDValue Val = Parts[0];
184
185 if (NumParts > 1) {
186 // Assemble the value from multiple parts.
187 if (ValueVT.isInteger()) {
188 unsigned PartBits = PartVT.getSizeInBits();
189 unsigned ValueBits = ValueVT.getSizeInBits();
190
191 // Assemble the power of 2 part.
192 unsigned RoundParts = llvm::bit_floor(NumParts);
193 unsigned RoundBits = PartBits * RoundParts;
194 EVT RoundVT = RoundBits == ValueBits ?
195 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
196 SDValue Lo, Hi;
197
198 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
199
200 if (RoundParts > 2) {
201 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
202 InChain);
203 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
204 PartVT, HalfVT, V, InChain);
205 } else {
206 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
207 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
208 }
209
210 if (DAG.getDataLayout().isBigEndian())
211 std::swap(Lo, Hi);
212
213 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
214
215 if (RoundParts < NumParts) {
216 // Assemble the trailing non-power-of-2 part.
217 unsigned OddParts = NumParts - RoundParts;
218 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
219 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
220 OddVT, V, InChain, CC);
221
222 // Combine the round and odd parts.
223 Lo = Val;
224 if (DAG.getDataLayout().isBigEndian())
225 std::swap(Lo, Hi);
226 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
227 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
228 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
229 DAG.getConstant(Lo.getValueSizeInBits(), DL,
231 TotalVT, DAG.getDataLayout())));
232 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
233 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
234 }
235 } else if (PartVT.isFloatingPoint()) {
236 // FP split into multiple FP parts (for ppcf128)
237 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
238 "Unexpected split");
239 SDValue Lo, Hi;
240 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
241 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
242 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
243 std::swap(Lo, Hi);
244 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
245 } else {
246 // FP split into integer parts (soft fp)
247 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
248 !PartVT.isVector() && "Unexpected split");
249 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
250 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
251 InChain, CC);
252 }
253 }
254
255 // There is now one part, held in Val. Correct it to match ValueVT.
256 // PartEVT is the type of the register class that holds the value.
257 // ValueVT is the type of the inline asm operation.
258 EVT PartEVT = Val.getValueType();
259
260 if (PartEVT == ValueVT)
261 return Val;
262
263 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
264 ValueVT.bitsLT(PartEVT)) {
265 // For an FP value in an integer part, we need to truncate to the right
266 // width first.
267 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
268 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
269 }
270
271 // Handle types that have the same size.
272 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
273 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
274
275 // Handle types with different sizes.
276 if (PartEVT.isInteger() && ValueVT.isInteger()) {
277 if (ValueVT.bitsLT(PartEVT)) {
278 // For a truncate, see if we have any information to
279 // indicate whether the truncated bits will always be
280 // zero or sign-extension.
281 if (AssertOp)
282 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
283 DAG.getValueType(ValueVT));
284 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
285 }
286 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
287 }
288
289 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
290 // FP_ROUND's are always exact here.
291 if (ValueVT.bitsLT(Val.getValueType())) {
292
293 SDValue NoChange =
295
297 llvm::Attribute::StrictFP)) {
298 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
299 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
300 NoChange);
301 }
302
303 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
304 }
305
306 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
307 }
308
309 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
310 // then truncating.
311 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
312 ValueVT.bitsLT(PartEVT)) {
313 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
314 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
315 }
316
317 report_fatal_error("Unknown mismatch in getCopyFromParts!");
318}
319
321 const Twine &ErrMsg) {
322 const Instruction *I = dyn_cast_or_null<Instruction>(V);
323 if (!V)
324 return Ctx.emitError(ErrMsg);
325
326 const char *AsmError = ", possible invalid constraint for vector type";
327 if (const CallInst *CI = dyn_cast<CallInst>(I))
328 if (CI->isInlineAsm())
329 return Ctx.emitError(I, ErrMsg + AsmError);
330
331 return Ctx.emitError(I, ErrMsg);
332}
333
334/// getCopyFromPartsVector - Create a value that contains the specified legal
335/// parts combined into the value they represent. If the parts combine to a
336/// type larger than ValueVT then AssertOp can be used to specify whether the
337/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
338/// ValueVT (ISD::AssertSext).
340 const SDValue *Parts, unsigned NumParts,
341 MVT PartVT, EVT ValueVT, const Value *V,
342 SDValue InChain,
343 std::optional<CallingConv::ID> CallConv) {
344 assert(ValueVT.isVector() && "Not a vector value");
345 assert(NumParts > 0 && "No parts to assemble!");
346 const bool IsABIRegCopy = CallConv.has_value();
347
348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
349 SDValue Val = Parts[0];
350
351 // Handle a multi-element vector.
352 if (NumParts > 1) {
353 EVT IntermediateVT;
354 MVT RegisterVT;
355 unsigned NumIntermediates;
356 unsigned NumRegs;
357
358 if (IsABIRegCopy) {
360 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
361 NumIntermediates, RegisterVT);
362 } else {
363 NumRegs =
364 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
365 NumIntermediates, RegisterVT);
366 }
367
368 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
369 NumParts = NumRegs; // Silence a compiler warning.
370 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
371 assert(RegisterVT.getSizeInBits() ==
372 Parts[0].getSimpleValueType().getSizeInBits() &&
373 "Part type sizes don't match!");
374
375 // Assemble the parts into intermediate operands.
376 SmallVector<SDValue, 8> Ops(NumIntermediates);
377 if (NumIntermediates == NumParts) {
378 // If the register was not expanded, truncate or copy the value,
379 // as appropriate.
380 for (unsigned i = 0; i != NumParts; ++i)
381 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
382 V, InChain, CallConv);
383 } else if (NumParts > 0) {
384 // If the intermediate type was expanded, build the intermediate
385 // operands from the parts.
386 assert(NumParts % NumIntermediates == 0 &&
387 "Must expand into a divisible number of parts!");
388 unsigned Factor = NumParts / NumIntermediates;
389 for (unsigned i = 0; i != NumIntermediates; ++i)
390 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
391 IntermediateVT, V, InChain, CallConv);
392 }
393
394 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
395 // intermediate operands.
396 EVT BuiltVectorTy =
397 IntermediateVT.isVector()
399 *DAG.getContext(), IntermediateVT.getScalarType(),
400 IntermediateVT.getVectorElementCount() * NumParts)
402 IntermediateVT.getScalarType(),
403 NumIntermediates);
404 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
406 DL, BuiltVectorTy, Ops);
407 }
408
409 // There is now one part, held in Val. Correct it to match ValueVT.
410 EVT PartEVT = Val.getValueType();
411
412 if (PartEVT == ValueVT)
413 return Val;
414
415 if (PartEVT.isVector()) {
416 // Vector/Vector bitcast.
417 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
418 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
419
420 // If the parts vector has more elements than the value vector, then we
421 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
422 // Extract the elements we want.
423 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
426 (PartEVT.getVectorElementCount().isScalable() ==
427 ValueVT.getVectorElementCount().isScalable()) &&
428 "Cannot narrow, it would be a lossy transformation");
429 PartEVT =
431 ValueVT.getVectorElementCount());
432 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
433 DAG.getVectorIdxConstant(0, DL));
434 if (PartEVT == ValueVT)
435 return Val;
436 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
437 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
438
439 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
440 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
441 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
442 }
443
444 // Promoted vector extract
445 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
446 }
447
448 // Trivial bitcast if the types are the same size and the destination
449 // vector type is legal.
450 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
451 TLI.isTypeLegal(ValueVT))
452 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
453
454 if (ValueVT.getVectorNumElements() != 1) {
455 // Certain ABIs require that vectors are passed as integers. For vectors
456 // are the same size, this is an obvious bitcast.
457 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
458 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
459 } else if (ValueVT.bitsLT(PartEVT)) {
460 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
461 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
462 // Drop the extra bits.
463 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
464 return DAG.getBitcast(ValueVT, Val);
465 }
466
468 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
469 return DAG.getUNDEF(ValueVT);
470 }
471
472 // Handle cases such as i8 -> <1 x i1>
473 EVT ValueSVT = ValueVT.getVectorElementType();
474 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
475 unsigned ValueSize = ValueSVT.getSizeInBits();
476 if (ValueSize == PartEVT.getSizeInBits()) {
477 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
478 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
479 // It's possible a scalar floating point type gets softened to integer and
480 // then promoted to a larger integer. If PartEVT is the larger integer
481 // we need to truncate it and then bitcast to the FP type.
482 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
483 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
484 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
485 Val = DAG.getBitcast(ValueSVT, Val);
486 } else {
487 Val = ValueVT.isFloatingPoint()
488 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
489 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
490 }
491 }
492
493 return DAG.getBuildVector(ValueVT, DL, Val);
494}
495
496static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
497 SDValue Val, SDValue *Parts, unsigned NumParts,
498 MVT PartVT, const Value *V,
499 std::optional<CallingConv::ID> CallConv);
500
501/// getCopyToParts - Create a series of nodes that contain the specified value
502/// split into legal parts. If the parts contain more bits than Val, then, for
503/// integers, ExtendKind can be used to specify how to generate the extra bits.
504static void
506 unsigned NumParts, MVT PartVT, const Value *V,
507 std::optional<CallingConv::ID> CallConv = std::nullopt,
508 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
509 // Let the target split the parts if it wants to
510 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
511 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
512 CallConv))
513 return;
514 EVT ValueVT = Val.getValueType();
515
516 // Handle the vector case separately.
517 if (ValueVT.isVector())
518 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
519 CallConv);
520
521 unsigned OrigNumParts = NumParts;
523 "Copying to an illegal type!");
524
525 if (NumParts == 0)
526 return;
527
528 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
529 EVT PartEVT = PartVT;
530 if (PartEVT == ValueVT) {
531 assert(NumParts == 1 && "No-op copy with multiple parts!");
532 Parts[0] = Val;
533 return;
534 }
535
536 unsigned PartBits = PartVT.getSizeInBits();
537 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
538 // If the parts cover more bits than the value has, promote the value.
539 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
540 assert(NumParts == 1 && "Do not know what to promote to!");
541 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
542 } else {
543 if (ValueVT.isFloatingPoint()) {
544 // FP values need to be bitcast, then extended if they are being put
545 // into a larger container.
546 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
547 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
548 }
549 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
550 ValueVT.isInteger() &&
551 "Unknown mismatch!");
552 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
553 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
554 if (PartVT == MVT::x86mmx)
555 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
556 }
557 } else if (PartBits == ValueVT.getSizeInBits()) {
558 // Different types of the same size.
559 assert(NumParts == 1 && PartEVT != ValueVT);
560 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
561 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
562 // If the parts cover less bits than value has, truncate the value.
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
567 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
570 }
571
572 // The value may have changed - recompute ValueVT.
573 ValueVT = Val.getValueType();
574 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
575 "Failed to tile the value with PartVT!");
576
577 if (NumParts == 1) {
578 if (PartEVT != ValueVT) {
580 "scalar-to-vector conversion failed");
581 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
582 }
583
584 Parts[0] = Val;
585 return;
586 }
587
588 // Expand the value into multiple parts.
589 if (NumParts & (NumParts - 1)) {
590 // The number of parts is not a power of 2. Split off and copy the tail.
591 assert(PartVT.isInteger() && ValueVT.isInteger() &&
592 "Do not know what to expand to!");
593 unsigned RoundParts = llvm::bit_floor(NumParts);
594 unsigned RoundBits = RoundParts * PartBits;
595 unsigned OddParts = NumParts - RoundParts;
596 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
597 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
598
599 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
600 CallConv);
601
602 if (DAG.getDataLayout().isBigEndian())
603 // The odd parts were reversed by getCopyToParts - unreverse them.
604 std::reverse(Parts + RoundParts, Parts + NumParts);
605
606 NumParts = RoundParts;
607 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
608 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
609 }
610
611 // The number of parts is a power of 2. Repeatedly bisect the value using
612 // EXTRACT_ELEMENT.
613 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
615 ValueVT.getSizeInBits()),
616 Val);
617
618 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
619 for (unsigned i = 0; i < NumParts; i += StepSize) {
620 unsigned ThisBits = StepSize * PartBits / 2;
621 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
622 SDValue &Part0 = Parts[i];
623 SDValue &Part1 = Parts[i+StepSize/2];
624
625 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
626 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
627 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
628 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
629
630 if (ThisBits == PartBits && ThisVT != PartVT) {
631 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
632 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
633 }
634 }
635 }
636
637 if (DAG.getDataLayout().isBigEndian())
638 std::reverse(Parts, Parts + OrigNumParts);
639}
640
642 const SDLoc &DL, EVT PartVT) {
643 if (!PartVT.isVector())
644 return SDValue();
645
646 EVT ValueVT = Val.getValueType();
647 EVT PartEVT = PartVT.getVectorElementType();
648 EVT ValueEVT = ValueVT.getVectorElementType();
649 ElementCount PartNumElts = PartVT.getVectorElementCount();
650 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
651
652 // We only support widening vectors with equivalent element types and
653 // fixed/scalable properties. If a target needs to widen a fixed-length type
654 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
655 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
656 PartNumElts.isScalable() != ValueNumElts.isScalable())
657 return SDValue();
658
659 // Have a try for bf16 because some targets share its ABI with fp16.
660 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
662 "Cannot widen to illegal type");
663 Val = DAG.getNode(ISD::BITCAST, DL,
664 ValueVT.changeVectorElementType(MVT::f16), Val);
665 } else if (PartEVT != ValueEVT) {
666 return SDValue();
667 }
668
669 // Widening a scalable vector to another scalable vector is done by inserting
670 // the vector into a larger undef one.
671 if (PartNumElts.isScalable())
672 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
673 Val, DAG.getVectorIdxConstant(0, DL));
674
675 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
676 // undef elements.
678 DAG.ExtractVectorElements(Val, Ops);
679 SDValue EltUndef = DAG.getUNDEF(PartEVT);
680 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
681
682 // FIXME: Use CONCAT for 2x -> 4x.
683 return DAG.getBuildVector(PartVT, DL, Ops);
684}
685
686/// getCopyToPartsVector - Create a series of nodes that contain the specified
687/// value split into legal parts.
688static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
689 SDValue Val, SDValue *Parts, unsigned NumParts,
690 MVT PartVT, const Value *V,
691 std::optional<CallingConv::ID> CallConv) {
692 EVT ValueVT = Val.getValueType();
693 assert(ValueVT.isVector() && "Not a vector");
694 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
695 const bool IsABIRegCopy = CallConv.has_value();
696
697 if (NumParts == 1) {
698 EVT PartEVT = PartVT;
699 if (PartEVT == ValueVT) {
700 // Nothing to do.
701 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
702 // Bitconvert vector->vector case.
703 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
704 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
705 Val = Widened;
706 } else if (PartVT.isVector() &&
708 ValueVT.getVectorElementType()) &&
709 PartEVT.getVectorElementCount() ==
710 ValueVT.getVectorElementCount()) {
711
712 // Promoted vector extract
713 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
714 } else if (PartEVT.isVector() &&
715 PartEVT.getVectorElementType() !=
716 ValueVT.getVectorElementType() &&
717 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
718 TargetLowering::TypeWidenVector) {
719 // Combination of widening and promotion.
720 EVT WidenVT =
722 PartVT.getVectorElementCount());
723 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
724 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
725 } else {
726 // Don't extract an integer from a float vector. This can happen if the
727 // FP type gets softened to integer and then promoted. The promotion
728 // prevents it from being picked up by the earlier bitcast case.
729 if (ValueVT.getVectorElementCount().isScalar() &&
730 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
731 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
732 DAG.getVectorIdxConstant(0, DL));
733 } else {
734 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
735 assert(PartVT.getFixedSizeInBits() > ValueSize &&
736 "lossy conversion of vector to scalar type");
737 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
738 Val = DAG.getBitcast(IntermediateType, Val);
739 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
740 }
741 }
742
743 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
744 Parts[0] = Val;
745 return;
746 }
747
748 // Handle a multi-element vector.
749 EVT IntermediateVT;
750 MVT RegisterVT;
751 unsigned NumIntermediates;
752 unsigned NumRegs;
753 if (IsABIRegCopy) {
755 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
756 RegisterVT);
757 } else {
758 NumRegs =
759 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
760 NumIntermediates, RegisterVT);
761 }
762
763 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
764 NumParts = NumRegs; // Silence a compiler warning.
765 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
766
767 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
768 "Mixing scalable and fixed vectors when copying in parts");
769
770 std::optional<ElementCount> DestEltCnt;
771
772 if (IntermediateVT.isVector())
773 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
774 else
775 DestEltCnt = ElementCount::getFixed(NumIntermediates);
776
777 EVT BuiltVectorTy = EVT::getVectorVT(
778 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
779
780 if (ValueVT == BuiltVectorTy) {
781 // Nothing to do.
782 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
783 // Bitconvert vector->vector case.
784 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
785 } else {
786 if (BuiltVectorTy.getVectorElementType().bitsGT(
787 ValueVT.getVectorElementType())) {
788 // Integer promotion.
789 ValueVT = EVT::getVectorVT(*DAG.getContext(),
790 BuiltVectorTy.getVectorElementType(),
791 ValueVT.getVectorElementCount());
792 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
793 }
794
795 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
796 Val = Widened;
797 }
798 }
799
800 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
801
802 // Split the vector into intermediate operands.
803 SmallVector<SDValue, 8> Ops(NumIntermediates);
804 for (unsigned i = 0; i != NumIntermediates; ++i) {
805 if (IntermediateVT.isVector()) {
806 // This does something sensible for scalable vectors - see the
807 // definition of EXTRACT_SUBVECTOR for further details.
808 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
809 Ops[i] =
810 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
811 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
812 } else {
813 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
814 DAG.getVectorIdxConstant(i, DL));
815 }
816 }
817
818 // Split the intermediate operands into legal parts.
819 if (NumParts == NumIntermediates) {
820 // If the register was not expanded, promote or copy the value,
821 // as appropriate.
822 for (unsigned i = 0; i != NumParts; ++i)
823 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
824 } else if (NumParts > 0) {
825 // If the intermediate type was expanded, split each the value into
826 // legal parts.
827 assert(NumIntermediates != 0 && "division by zero");
828 assert(NumParts % NumIntermediates == 0 &&
829 "Must expand into a divisible number of parts!");
830 unsigned Factor = NumParts / NumIntermediates;
831 for (unsigned i = 0; i != NumIntermediates; ++i)
832 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
833 CallConv);
834 }
835}
836
838 EVT valuevt, std::optional<CallingConv::ID> CC)
839 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
840 RegCount(1, regs.size()), CallConv(CC) {}
841
843 const DataLayout &DL, unsigned Reg, Type *Ty,
844 std::optional<CallingConv::ID> CC) {
845 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
846
847 CallConv = CC;
848
849 for (EVT ValueVT : ValueVTs) {
850 unsigned NumRegs =
853 : TLI.getNumRegisters(Context, ValueVT);
854 MVT RegisterVT =
857 : TLI.getRegisterType(Context, ValueVT);
858 for (unsigned i = 0; i != NumRegs; ++i)
859 Regs.push_back(Reg + i);
860 RegVTs.push_back(RegisterVT);
861 RegCount.push_back(NumRegs);
862 Reg += NumRegs;
863 }
864}
865
867 FunctionLoweringInfo &FuncInfo,
868 const SDLoc &dl, SDValue &Chain,
869 SDValue *Glue, const Value *V) const {
870 // A Value with type {} or [0 x %t] needs no registers.
871 if (ValueVTs.empty())
872 return SDValue();
873
874 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
875
876 // Assemble the legal parts into the final values.
877 SmallVector<SDValue, 4> Values(ValueVTs.size());
879 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
880 // Copy the legal parts from the registers.
881 EVT ValueVT = ValueVTs[Value];
882 unsigned NumRegs = RegCount[Value];
883 MVT RegisterVT = isABIMangled()
885 *DAG.getContext(), *CallConv, RegVTs[Value])
886 : RegVTs[Value];
887
888 Parts.resize(NumRegs);
889 for (unsigned i = 0; i != NumRegs; ++i) {
890 SDValue P;
891 if (!Glue) {
892 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
893 } else {
894 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
895 *Glue = P.getValue(2);
896 }
897
898 Chain = P.getValue(1);
899 Parts[i] = P;
900
901 // If the source register was virtual and if we know something about it,
902 // add an assert node.
903 if (!Register::isVirtualRegister(Regs[Part + i]) ||
904 !RegisterVT.isInteger())
905 continue;
906
908 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
909 if (!LOI)
910 continue;
911
912 unsigned RegSize = RegisterVT.getScalarSizeInBits();
913 unsigned NumSignBits = LOI->NumSignBits;
914 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
915
916 if (NumZeroBits == RegSize) {
917 // The current value is a zero.
918 // Explicitly express that as it would be easier for
919 // optimizations to kick in.
920 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
921 continue;
922 }
923
924 // FIXME: We capture more information than the dag can represent. For
925 // now, just use the tightest assertzext/assertsext possible.
926 bool isSExt;
927 EVT FromVT(MVT::Other);
928 if (NumZeroBits) {
929 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
930 isSExt = false;
931 } else if (NumSignBits > 1) {
932 FromVT =
933 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
934 isSExt = true;
935 } else {
936 continue;
937 }
938 // Add an assertion node.
939 assert(FromVT != MVT::Other);
940 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
941 RegisterVT, P, DAG.getValueType(FromVT));
942 }
943
944 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
945 RegisterVT, ValueVT, V, Chain, CallConv);
946 Part += NumRegs;
947 Parts.clear();
948 }
949
950 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
951}
952
954 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
955 const Value *V,
956 ISD::NodeType PreferredExtendType) const {
957 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
958 ISD::NodeType ExtendKind = PreferredExtendType;
959
960 // Get the list of the values's legal parts.
961 unsigned NumRegs = Regs.size();
962 SmallVector<SDValue, 8> Parts(NumRegs);
963 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
964 unsigned NumParts = RegCount[Value];
965
966 MVT RegisterVT = isABIMangled()
968 *DAG.getContext(), *CallConv, RegVTs[Value])
969 : RegVTs[Value];
970
971 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
972 ExtendKind = ISD::ZERO_EXTEND;
973
974 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
975 NumParts, RegisterVT, V, CallConv, ExtendKind);
976 Part += NumParts;
977 }
978
979 // Copy the parts into the registers.
980 SmallVector<SDValue, 8> Chains(NumRegs);
981 for (unsigned i = 0; i != NumRegs; ++i) {
982 SDValue Part;
983 if (!Glue) {
984 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
985 } else {
986 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
987 *Glue = Part.getValue(1);
988 }
989
990 Chains[i] = Part.getValue(0);
991 }
992
993 if (NumRegs == 1 || Glue)
994 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
995 // flagged to it. That is the CopyToReg nodes and the user are considered
996 // a single scheduling unit. If we create a TokenFactor and return it as
997 // chain, then the TokenFactor is both a predecessor (operand) of the
998 // user as well as a successor (the TF operands are flagged to the user).
999 // c1, f1 = CopyToReg
1000 // c2, f2 = CopyToReg
1001 // c3 = TokenFactor c1, c2
1002 // ...
1003 // = op c3, ..., f2
1004 Chain = Chains[NumRegs-1];
1005 else
1006 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1007}
1008
1010 unsigned MatchingIdx, const SDLoc &dl,
1011 SelectionDAG &DAG,
1012 std::vector<SDValue> &Ops) const {
1013 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1014
1015 InlineAsm::Flag Flag(Code, Regs.size());
1016 if (HasMatching)
1017 Flag.setMatchingOp(MatchingIdx);
1018 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
1019 // Put the register class of the virtual registers in the flag word. That
1020 // way, later passes can recompute register class constraints for inline
1021 // assembly as well as normal instructions.
1022 // Don't do this for tied operands that can use the regclass information
1023 // from the def.
1025 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1026 Flag.setRegClass(RC->getID());
1027 }
1028
1029 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1030 Ops.push_back(Res);
1031
1032 if (Code == InlineAsm::Kind::Clobber) {
1033 // Clobbers should always have a 1:1 mapping with registers, and may
1034 // reference registers that have illegal (e.g. vector) types. Hence, we
1035 // shouldn't try to apply any sort of splitting logic to them.
1036 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1037 "No 1:1 mapping from clobbers to regs?");
1039 (void)SP;
1040 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1041 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1042 assert(
1043 (Regs[I] != SP ||
1045 "If we clobbered the stack pointer, MFI should know about it.");
1046 }
1047 return;
1048 }
1049
1050 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1051 MVT RegisterVT = RegVTs[Value];
1052 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1053 RegisterVT);
1054 for (unsigned i = 0; i != NumRegs; ++i) {
1055 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1056 unsigned TheReg = Regs[Reg++];
1057 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1058 }
1059 }
1060}
1061
1065 unsigned I = 0;
1066 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1067 unsigned RegCount = std::get<0>(CountAndVT);
1068 MVT RegisterVT = std::get<1>(CountAndVT);
1069 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1070 for (unsigned E = I + RegCount; I != E; ++I)
1071 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1072 }
1073 return OutVec;
1074}
1075
1077 AssumptionCache *ac,
1078 const TargetLibraryInfo *li) {
1079 AA = aa;
1080 AC = ac;
1081 GFI = gfi;
1082 LibInfo = li;
1084 LPadToCallSiteMap.clear();
1086 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1088}
1089
1091 NodeMap.clear();
1092 UnusedArgNodeMap.clear();
1093 PendingLoads.clear();
1094 PendingExports.clear();
1095 PendingConstrainedFP.clear();
1096 PendingConstrainedFPStrict.clear();
1097 CurInst = nullptr;
1098 HasTailCall = false;
1099 SDNodeOrder = LowestSDNodeOrder;
1101}
1102
1104 DanglingDebugInfoMap.clear();
1105}
1106
1107// Update DAG root to include dependencies on Pending chains.
1108SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1109 SDValue Root = DAG.getRoot();
1110
1111 if (Pending.empty())
1112 return Root;
1113
1114 // Add current root to PendingChains, unless we already indirectly
1115 // depend on it.
1116 if (Root.getOpcode() != ISD::EntryToken) {
1117 unsigned i = 0, e = Pending.size();
1118 for (; i != e; ++i) {
1119 assert(Pending[i].getNode()->getNumOperands() > 1);
1120 if (Pending[i].getNode()->getOperand(0) == Root)
1121 break; // Don't add the root if we already indirectly depend on it.
1122 }
1123
1124 if (i == e)
1125 Pending.push_back(Root);
1126 }
1127
1128 if (Pending.size() == 1)
1129 Root = Pending[0];
1130 else
1131 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1132
1133 DAG.setRoot(Root);
1134 Pending.clear();
1135 return Root;
1136}
1137
1139 return updateRoot(PendingLoads);
1140}
1141
1143 // Chain up all pending constrained intrinsics together with all
1144 // pending loads, by simply appending them to PendingLoads and
1145 // then calling getMemoryRoot().
1146 PendingLoads.reserve(PendingLoads.size() +
1147 PendingConstrainedFP.size() +
1148 PendingConstrainedFPStrict.size());
1149 PendingLoads.append(PendingConstrainedFP.begin(),
1150 PendingConstrainedFP.end());
1151 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1152 PendingConstrainedFPStrict.end());
1153 PendingConstrainedFP.clear();
1154 PendingConstrainedFPStrict.clear();
1155 return getMemoryRoot();
1156}
1157
1159 // We need to emit pending fpexcept.strict constrained intrinsics,
1160 // so append them to the PendingExports list.
1161 PendingExports.append(PendingConstrainedFPStrict.begin(),
1162 PendingConstrainedFPStrict.end());
1163 PendingConstrainedFPStrict.clear();
1164 return updateRoot(PendingExports);
1165}
1166
1168 DILocalVariable *Variable,
1170 DebugLoc DL) {
1171 assert(Variable && "Missing variable");
1172
1173 // Check if address has undef value.
1174 if (!Address || isa<UndefValue>(Address) ||
1175 (Address->use_empty() && !isa<Argument>(Address))) {
1176 LLVM_DEBUG(
1177 dbgs()
1178 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1179 return;
1180 }
1181
1182 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1183
1184 SDValue &N = NodeMap[Address];
1185 if (!N.getNode() && isa<Argument>(Address))
1186 // Check unused arguments map.
1187 N = UnusedArgNodeMap[Address];
1188 SDDbgValue *SDV;
1189 if (N.getNode()) {
1190 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1191 Address = BCI->getOperand(0);
1192 // Parameters are handled specially.
1193 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1194 if (IsParameter && FINode) {
1195 // Byval parameter. We have a frame index at this point.
1196 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1197 /*IsIndirect*/ true, DL, SDNodeOrder);
1198 } else if (isa<Argument>(Address)) {
1199 // Address is an argument, so try to emit its dbg value using
1200 // virtual register info from the FuncInfo.ValueMap.
1201 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1202 FuncArgumentDbgValueKind::Declare, N);
1203 return;
1204 } else {
1205 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1206 true, DL, SDNodeOrder);
1207 }
1208 DAG.AddDbgValue(SDV, IsParameter);
1209 } else {
1210 // If Address is an argument then try to emit its dbg value using
1211 // virtual register info from the FuncInfo.ValueMap.
1212 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1213 FuncArgumentDbgValueKind::Declare, N)) {
1214 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1215 << " (could not emit func-arg dbg_value)\n");
1216 }
1217 }
1218 return;
1219}
1220
1222 // Add SDDbgValue nodes for any var locs here. Do so before updating
1223 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1224 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1225 // Add SDDbgValue nodes for any var locs here. Do so before updating
1226 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1227 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1228 It != End; ++It) {
1229 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1230 dropDanglingDebugInfo(Var, It->Expr);
1231 if (It->Values.isKillLocation(It->Expr)) {
1232 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1233 continue;
1234 }
1235 SmallVector<Value *> Values(It->Values.location_ops());
1236 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1237 It->Values.hasArgList())) {
1239 for (Value *V : It->Values.location_ops())
1240 Vals.push_back(V);
1242 FnVarLocs->getDILocalVariable(It->VariableID),
1243 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1244 }
1245 }
1246 }
1247
1248 // We must skip DPValues if they've already been processed above as we
1249 // have just emitted the debug values resulting from assignment tracking
1250 // analysis, making any existing DPValues redundant (and probably less
1251 // correct). We still need to process DPLabels. This does sink DPLabels
1252 // to the bottom of the group of debug records. That sholdn't be important
1253 // as it does so deterministcally and ordering between DPLabels and DPValues
1254 // is immaterial (other than for MIR/IR printing).
1255 bool SkipDPValues = DAG.getFunctionVarLocs();
1256 // Is there is any debug-info attached to this instruction, in the form of
1257 // DbgRecord non-instruction debug-info records.
1258 for (DbgRecord &DR : I.getDbgRecordRange()) {
1259 if (DPLabel *DPL = dyn_cast<DPLabel>(&DR)) {
1260 assert(DPL->getLabel() && "Missing label");
1261 SDDbgLabel *SDV =
1262 DAG.getDbgLabel(DPL->getLabel(), DPL->getDebugLoc(), SDNodeOrder);
1263 DAG.AddDbgLabel(SDV);
1264 continue;
1265 }
1266
1267 if (SkipDPValues)
1268 continue;
1269 DPValue &DPV = cast<DPValue>(DR);
1270 DILocalVariable *Variable = DPV.getVariable();
1273
1275 if (FuncInfo.PreprocessedDPVDeclares.contains(&DPV))
1276 continue;
1277 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DPV
1278 << "\n");
1280 DPV.getDebugLoc());
1281 continue;
1282 }
1283
1284 // A DPValue with no locations is a kill location.
1286 if (Values.empty()) {
1288 SDNodeOrder);
1289 continue;
1290 }
1291
1292 // A DPValue with an undef or absent location is also a kill location.
1293 if (llvm::any_of(Values,
1294 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1296 SDNodeOrder);
1297 continue;
1298 }
1299
1300 bool IsVariadic = DPV.hasArgList();
1301 if (!handleDebugValue(Values, Variable, Expression, DPV.getDebugLoc(),
1302 SDNodeOrder, IsVariadic)) {
1303 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1304 DPV.getDebugLoc(), SDNodeOrder);
1305 }
1306 }
1307}
1308
1310 visitDbgInfo(I);
1311
1312 // Set up outgoing PHI node register values before emitting the terminator.
1313 if (I.isTerminator()) {
1314 HandlePHINodesInSuccessorBlocks(I.getParent());
1315 }
1316
1317 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1318 if (!isa<DbgInfoIntrinsic>(I))
1319 ++SDNodeOrder;
1320
1321 CurInst = &I;
1322
1323 // Set inserted listener only if required.
1324 bool NodeInserted = false;
1325 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1326 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1327 if (PCSectionsMD) {
1328 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1329 DAG, [&](SDNode *) { NodeInserted = true; });
1330 }
1331
1332 visit(I.getOpcode(), I);
1333
1334 if (!I.isTerminator() && !HasTailCall &&
1335 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1337
1338 // Handle metadata.
1339 if (PCSectionsMD) {
1340 auto It = NodeMap.find(&I);
1341 if (It != NodeMap.end()) {
1342 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1343 } else if (NodeInserted) {
1344 // This should not happen; if it does, don't let it go unnoticed so we can
1345 // fix it. Relevant visit*() function is probably missing a setValue().
1346 errs() << "warning: loosing !pcsections metadata ["
1347 << I.getModule()->getName() << "]\n";
1348 LLVM_DEBUG(I.dump());
1349 assert(false);
1350 }
1351 }
1352
1353 CurInst = nullptr;
1354}
1355
1356void SelectionDAGBuilder::visitPHI(const PHINode &) {
1357 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1358}
1359
1360void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1361 // Note: this doesn't use InstVisitor, because it has to work with
1362 // ConstantExpr's in addition to instructions.
1363 switch (Opcode) {
1364 default: llvm_unreachable("Unknown instruction type encountered!");
1365 // Build the switch statement using the Instruction.def file.
1366#define HANDLE_INST(NUM, OPCODE, CLASS) \
1367 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1368#include "llvm/IR/Instruction.def"
1369 }
1370}
1371
1373 DILocalVariable *Variable,
1374 DebugLoc DL, unsigned Order,
1377 // For variadic dbg_values we will now insert an undef.
1378 // FIXME: We can potentially recover these!
1380 for (const Value *V : Values) {
1381 auto *Undef = UndefValue::get(V->getType());
1383 }
1384 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1385 /*IsIndirect=*/false, DL, Order,
1386 /*IsVariadic=*/true);
1387 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1388 return true;
1389}
1390
1392 DILocalVariable *Var,
1393 DIExpression *Expr,
1394 bool IsVariadic, DebugLoc DL,
1395 unsigned Order) {
1396 if (IsVariadic) {
1397 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1398 return;
1399 }
1400 // TODO: Dangling debug info will eventually either be resolved or produce
1401 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1402 // between the original dbg.value location and its resolved DBG_VALUE,
1403 // which we should ideally fill with an extra Undef DBG_VALUE.
1404 assert(Values.size() == 1);
1405 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1406}
1407
1409 const DIExpression *Expr) {
1410 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1411 DIVariable *DanglingVariable = DDI.getVariable();
1412 DIExpression *DanglingExpr = DDI.getExpression();
1413 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1414 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1415 << printDDI(nullptr, DDI) << "\n");
1416 return true;
1417 }
1418 return false;
1419 };
1420
1421 for (auto &DDIMI : DanglingDebugInfoMap) {
1422 DanglingDebugInfoVector &DDIV = DDIMI.second;
1423
1424 // If debug info is to be dropped, run it through final checks to see
1425 // whether it can be salvaged.
1426 for (auto &DDI : DDIV)
1427 if (isMatchingDbgValue(DDI))
1428 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1429
1430 erase_if(DDIV, isMatchingDbgValue);
1431 }
1432}
1433
1434// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1435// generate the debug data structures now that we've seen its definition.
1437 SDValue Val) {
1438 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1439 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1440 return;
1441
1442 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1443 for (auto &DDI : DDIV) {
1444 DebugLoc DL = DDI.getDebugLoc();
1445 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1446 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1447 DILocalVariable *Variable = DDI.getVariable();
1448 DIExpression *Expr = DDI.getExpression();
1450 "Expected inlined-at fields to agree");
1451 SDDbgValue *SDV;
1452 if (Val.getNode()) {
1453 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1454 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1455 // we couldn't resolve it directly when examining the DbgValue intrinsic
1456 // in the first place we should not be more successful here). Unless we
1457 // have some test case that prove this to be correct we should avoid
1458 // calling EmitFuncArgumentDbgValue here.
1459 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1460 FuncArgumentDbgValueKind::Value, Val)) {
1461 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1462 << printDDI(V, DDI) << "\n");
1463 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1464 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1465 // inserted after the definition of Val when emitting the instructions
1466 // after ISel. An alternative could be to teach
1467 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1468 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1469 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1470 << ValSDNodeOrder << "\n");
1471 SDV = getDbgValue(Val, Variable, Expr, DL,
1472 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1473 DAG.AddDbgValue(SDV, false);
1474 } else
1475 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1476 << printDDI(V, DDI)
1477 << " in EmitFuncArgumentDbgValue\n");
1478 } else {
1479 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1480 << "\n");
1481 auto Undef = UndefValue::get(V->getType());
1482 auto SDV =
1483 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1484 DAG.AddDbgValue(SDV, false);
1485 }
1486 }
1487 DDIV.clear();
1488}
1489
1491 DanglingDebugInfo &DDI) {
1492 // TODO: For the variadic implementation, instead of only checking the fail
1493 // state of `handleDebugValue`, we need know specifically which values were
1494 // invalid, so that we attempt to salvage only those values when processing
1495 // a DIArgList.
1496 const Value *OrigV = V;
1497 DILocalVariable *Var = DDI.getVariable();
1498 DIExpression *Expr = DDI.getExpression();
1499 DebugLoc DL = DDI.getDebugLoc();
1500 unsigned SDOrder = DDI.getSDNodeOrder();
1501
1502 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1503 // that DW_OP_stack_value is desired.
1504 bool StackValue = true;
1505
1506 // Can this Value can be encoded without any further work?
1507 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1508 return;
1509
1510 // Attempt to salvage back through as many instructions as possible. Bail if
1511 // a non-instruction is seen, such as a constant expression or global
1512 // variable. FIXME: Further work could recover those too.
1513 while (isa<Instruction>(V)) {
1514 const Instruction &VAsInst = *cast<const Instruction>(V);
1515 // Temporary "0", awaiting real implementation.
1517 SmallVector<Value *, 4> AdditionalValues;
1518 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1519 Expr->getNumLocationOperands(), Ops,
1520 AdditionalValues);
1521 // If we cannot salvage any further, and haven't yet found a suitable debug
1522 // expression, bail out.
1523 if (!V)
1524 break;
1525
1526 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1527 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1528 // here for variadic dbg_values, remove that condition.
1529 if (!AdditionalValues.empty())
1530 break;
1531
1532 // New value and expr now represent this debuginfo.
1533 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1534
1535 // Some kind of simplification occurred: check whether the operand of the
1536 // salvaged debug expression can be encoded in this DAG.
1537 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1538 LLVM_DEBUG(
1539 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1540 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1541 return;
1542 }
1543 }
1544
1545 // This was the final opportunity to salvage this debug information, and it
1546 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1547 // any earlier variable location.
1548 assert(OrigV && "V shouldn't be null");
1549 auto *Undef = UndefValue::get(OrigV->getType());
1550 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1551 DAG.AddDbgValue(SDV, false);
1552 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1553 << printDDI(OrigV, DDI) << "\n");
1554}
1555
1557 DIExpression *Expr,
1558 DebugLoc DbgLoc,
1559 unsigned Order) {
1563 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1564 /*IsVariadic*/ false);
1565}
1566
1568 DILocalVariable *Var,
1569 DIExpression *Expr, DebugLoc DbgLoc,
1570 unsigned Order, bool IsVariadic) {
1571 if (Values.empty())
1572 return true;
1573
1574 // Filter EntryValue locations out early.
1575 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1576 return true;
1577
1578 SmallVector<SDDbgOperand> LocationOps;
1579 SmallVector<SDNode *> Dependencies;
1580 for (const Value *V : Values) {
1581 // Constant value.
1582 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1583 isa<ConstantPointerNull>(V)) {
1584 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1585 continue;
1586 }
1587
1588 // Look through IntToPtr constants.
1589 if (auto *CE = dyn_cast<ConstantExpr>(V))
1590 if (CE->getOpcode() == Instruction::IntToPtr) {
1591 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1592 continue;
1593 }
1594
1595 // If the Value is a frame index, we can create a FrameIndex debug value
1596 // without relying on the DAG at all.
1597 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1598 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1599 if (SI != FuncInfo.StaticAllocaMap.end()) {
1600 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1601 continue;
1602 }
1603 }
1604
1605 // Do not use getValue() in here; we don't want to generate code at
1606 // this point if it hasn't been done yet.
1607 SDValue N = NodeMap[V];
1608 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1609 N = UnusedArgNodeMap[V];
1610 if (N.getNode()) {
1611 // Only emit func arg dbg value for non-variadic dbg.values for now.
1612 if (!IsVariadic &&
1613 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1614 FuncArgumentDbgValueKind::Value, N))
1615 return true;
1616 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1617 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1618 // describe stack slot locations.
1619 //
1620 // Consider "int x = 0; int *px = &x;". There are two kinds of
1621 // interesting debug values here after optimization:
1622 //
1623 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1624 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1625 //
1626 // Both describe the direct values of their associated variables.
1627 Dependencies.push_back(N.getNode());
1628 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1629 continue;
1630 }
1631 LocationOps.emplace_back(
1632 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1633 continue;
1634 }
1635
1637 // Special rules apply for the first dbg.values of parameter variables in a
1638 // function. Identify them by the fact they reference Argument Values, that
1639 // they're parameters, and they are parameters of the current function. We
1640 // need to let them dangle until they get an SDNode.
1641 bool IsParamOfFunc =
1642 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1643 if (IsParamOfFunc)
1644 return false;
1645
1646 // The value is not used in this block yet (or it would have an SDNode).
1647 // We still want the value to appear for the user if possible -- if it has
1648 // an associated VReg, we can refer to that instead.
1649 auto VMI = FuncInfo.ValueMap.find(V);
1650 if (VMI != FuncInfo.ValueMap.end()) {
1651 unsigned Reg = VMI->second;
1652 // If this is a PHI node, it may be split up into several MI PHI nodes
1653 // (in FunctionLoweringInfo::set).
1654 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1655 V->getType(), std::nullopt);
1656 if (RFV.occupiesMultipleRegs()) {
1657 // FIXME: We could potentially support variadic dbg_values here.
1658 if (IsVariadic)
1659 return false;
1660 unsigned Offset = 0;
1661 unsigned BitsToDescribe = 0;
1662 if (auto VarSize = Var->getSizeInBits())
1663 BitsToDescribe = *VarSize;
1664 if (auto Fragment = Expr->getFragmentInfo())
1665 BitsToDescribe = Fragment->SizeInBits;
1666 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1667 // Bail out if all bits are described already.
1668 if (Offset >= BitsToDescribe)
1669 break;
1670 // TODO: handle scalable vectors.
1671 unsigned RegisterSize = RegAndSize.second;
1672 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1673 ? BitsToDescribe - Offset
1674 : RegisterSize;
1675 auto FragmentExpr = DIExpression::createFragmentExpression(
1676 Expr, Offset, FragmentSize);
1677 if (!FragmentExpr)
1678 continue;
1680 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
1681 DAG.AddDbgValue(SDV, false);
1682 Offset += RegisterSize;
1683 }
1684 return true;
1685 }
1686 // We can use simple vreg locations for variadic dbg_values as well.
1687 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1688 continue;
1689 }
1690 // We failed to create a SDDbgOperand for V.
1691 return false;
1692 }
1693
1694 // We have created a SDDbgOperand for each Value in Values.
1695 // Should use Order instead of SDNodeOrder?
1696 assert(!LocationOps.empty());
1697 SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1698 /*IsIndirect=*/false, DbgLoc,
1699 SDNodeOrder, IsVariadic);
1700 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1701 return true;
1702}
1703
1705 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1706 for (auto &Pair : DanglingDebugInfoMap)
1707 for (auto &DDI : Pair.second)
1708 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1710}
1711
1712/// getCopyFromRegs - If there was virtual register allocated for the value V
1713/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1716 SDValue Result;
1717
1718 if (It != FuncInfo.ValueMap.end()) {
1719 Register InReg = It->second;
1720
1722 DAG.getDataLayout(), InReg, Ty,
1723 std::nullopt); // This is not an ABI copy.
1724 SDValue Chain = DAG.getEntryNode();
1725 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1726 V);
1727 resolveDanglingDebugInfo(V, Result);
1728 }
1729
1730 return Result;
1731}
1732
1733/// getValue - Return an SDValue for the given Value.
1735 // If we already have an SDValue for this value, use it. It's important
1736 // to do this first, so that we don't create a CopyFromReg if we already
1737 // have a regular SDValue.
1738 SDValue &N = NodeMap[V];
1739 if (N.getNode()) return N;
1740
1741 // If there's a virtual register allocated and initialized for this
1742 // value, use it.
1743 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1744 return copyFromReg;
1745
1746 // Otherwise create a new SDValue and remember it.
1747 SDValue Val = getValueImpl(V);
1748 NodeMap[V] = Val;
1750 return Val;
1751}
1752
1753/// getNonRegisterValue - Return an SDValue for the given Value, but
1754/// don't look in FuncInfo.ValueMap for a virtual register.
1756 // If we already have an SDValue for this value, use it.
1757 SDValue &N = NodeMap[V];
1758 if (N.getNode()) {
1759 if (isIntOrFPConstant(N)) {
1760 // Remove the debug location from the node as the node is about to be used
1761 // in a location which may differ from the original debug location. This
1762 // is relevant to Constant and ConstantFP nodes because they can appear
1763 // as constant expressions inside PHI nodes.
1764 N->setDebugLoc(DebugLoc());
1765 }
1766 return N;
1767 }
1768
1769 // Otherwise create a new SDValue and remember it.
1770 SDValue Val = getValueImpl(V);
1771 NodeMap[V] = Val;
1773 return Val;
1774}
1775
1776/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1777/// Create an SDValue for the given value.
1780
1781 if (const Constant *C = dyn_cast<Constant>(V)) {
1782 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1783
1784 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1785 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1786
1787 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1788 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1789
1790 if (isa<ConstantPointerNull>(C)) {
1791 unsigned AS = V->getType()->getPointerAddressSpace();
1792 return DAG.getConstant(0, getCurSDLoc(),
1793 TLI.getPointerTy(DAG.getDataLayout(), AS));
1794 }
1795
1796 if (match(C, m_VScale()))
1797 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1798
1799 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1800 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1801
1802 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1803 return DAG.getUNDEF(VT);
1804
1805 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1806 visit(CE->getOpcode(), *CE);
1807 SDValue N1 = NodeMap[V];
1808 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1809 return N1;
1810 }
1811
1812 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1814 for (const Use &U : C->operands()) {
1815 SDNode *Val = getValue(U).getNode();
1816 // If the operand is an empty aggregate, there are no values.
1817 if (!Val) continue;
1818 // Add each leaf value from the operand to the Constants list
1819 // to form a flattened list of all the values.
1820 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1821 Constants.push_back(SDValue(Val, i));
1822 }
1823
1825 }
1826
1827 if (const ConstantDataSequential *CDS =
1828 dyn_cast<ConstantDataSequential>(C)) {
1830 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1831 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1832 // Add each leaf value from the operand to the Constants list
1833 // to form a flattened list of all the values.
1834 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1835 Ops.push_back(SDValue(Val, i));
1836 }
1837
1838 if (isa<ArrayType>(CDS->getType()))
1839 return DAG.getMergeValues(Ops, getCurSDLoc());
1840 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1841 }
1842
1843 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1844 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1845 "Unknown struct or array constant!");
1846
1847 SmallVector<EVT, 4> ValueVTs;
1848 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1849 unsigned NumElts = ValueVTs.size();
1850 if (NumElts == 0)
1851 return SDValue(); // empty struct
1853 for (unsigned i = 0; i != NumElts; ++i) {
1854 EVT EltVT = ValueVTs[i];
1855 if (isa<UndefValue>(C))
1856 Constants[i] = DAG.getUNDEF(EltVT);
1857 else if (EltVT.isFloatingPoint())
1858 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1859 else
1860 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1861 }
1862
1864 }
1865
1866 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1867 return DAG.getBlockAddress(BA, VT);
1868
1869 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1870 return getValue(Equiv->getGlobalValue());
1871
1872 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1873 return getValue(NC->getGlobalValue());
1874
1875 if (VT == MVT::aarch64svcount) {
1876 assert(C->isNullValue() && "Can only zero this target type!");
1877 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1878 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1879 }
1880
1881 VectorType *VecTy = cast<VectorType>(V->getType());
1882
1883 // Now that we know the number and type of the elements, get that number of
1884 // elements into the Ops array based on what kind of constant it is.
1885 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1887 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1888 for (unsigned i = 0; i != NumElements; ++i)
1889 Ops.push_back(getValue(CV->getOperand(i)));
1890
1891 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1892 }
1893
1894 if (isa<ConstantAggregateZero>(C)) {
1895 EVT EltVT =
1897
1898 SDValue Op;
1899 if (EltVT.isFloatingPoint())
1900 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1901 else
1902 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1903
1904 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1905 }
1906
1907 llvm_unreachable("Unknown vector constant");
1908 }
1909
1910 // If this is a static alloca, generate it as the frameindex instead of
1911 // computation.
1912 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1914 FuncInfo.StaticAllocaMap.find(AI);
1915 if (SI != FuncInfo.StaticAllocaMap.end())
1916 return DAG.getFrameIndex(
1917 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1918 }
1919
1920 // If this is an instruction which fast-isel has deferred, select it now.
1921 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1923
1924 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1925 Inst->getType(), std::nullopt);
1926 SDValue Chain = DAG.getEntryNode();
1927 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1928 }
1929
1930 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1931 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1932
1933 if (const auto *BB = dyn_cast<BasicBlock>(V))
1934 return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1935
1936 llvm_unreachable("Can't get register for value!");
1937}
1938
1939void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1941 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1942 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1943 bool IsSEH = isAsynchronousEHPersonality(Pers);
1944 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1945 if (!IsSEH)
1946 CatchPadMBB->setIsEHScopeEntry();
1947 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1948 if (IsMSVCCXX || IsCoreCLR)
1949 CatchPadMBB->setIsEHFuncletEntry();
1950}
1951
1952void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1953 // Update machine-CFG edge.
1954 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1955 FuncInfo.MBB->addSuccessor(TargetMBB);
1956 TargetMBB->setIsEHCatchretTarget(true);
1958
1960 bool IsSEH = isAsynchronousEHPersonality(Pers);
1961 if (IsSEH) {
1962 // If this is not a fall-through branch or optimizations are switched off,
1963 // emit the branch.
1964 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1966 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1967 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1968 return;
1969 }
1970
1971 // Figure out the funclet membership for the catchret's successor.
1972 // This will be used by the FuncletLayout pass to determine how to order the
1973 // BB's.
1974 // A 'catchret' returns to the outer scope's color.
1975 Value *ParentPad = I.getCatchSwitchParentPad();
1976 const BasicBlock *SuccessorColor;
1977 if (isa<ConstantTokenNone>(ParentPad))
1978 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1979 else
1980 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1981 assert(SuccessorColor && "No parent funclet for catchret!");
1982 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1983 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1984
1985 // Create the terminator node.
1987 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1988 DAG.getBasicBlock(SuccessorColorMBB));
1989 DAG.setRoot(Ret);
1990}
1991
1992void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1993 // Don't emit any special code for the cleanuppad instruction. It just marks
1994 // the start of an EH scope/funclet.
1997 if (Pers != EHPersonality::Wasm_CXX) {
2000 }
2001}
2002
2003// In wasm EH, even though a catchpad may not catch an exception if a tag does
2004// not match, it is OK to add only the first unwind destination catchpad to the
2005// successors, because there will be at least one invoke instruction within the
2006// catch scope that points to the next unwind destination, if one exists, so
2007// CFGSort cannot mess up with BB sorting order.
2008// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
2009// call within them, and catchpads only consisting of 'catch (...)' have a
2010// '__cxa_end_catch' call within them, both of which generate invokes in case
2011// the next unwind destination exists, i.e., the next unwind destination is not
2012// the caller.)
2013//
2014// Having at most one EH pad successor is also simpler and helps later
2015// transformations.
2016//
2017// For example,
2018// current:
2019// invoke void @foo to ... unwind label %catch.dispatch
2020// catch.dispatch:
2021// %0 = catchswitch within ... [label %catch.start] unwind label %next
2022// catch.start:
2023// ...
2024// ... in this BB or some other child BB dominated by this BB there will be an
2025// invoke that points to 'next' BB as an unwind destination
2026//
2027// next: ; We don't need to add this to 'current' BB's successor
2028// ...
2030 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2031 BranchProbability Prob,
2032 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2033 &UnwindDests) {
2034 while (EHPadBB) {
2035 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2036 if (isa<CleanupPadInst>(Pad)) {
2037 // Stop on cleanup pads.
2038 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2039 UnwindDests.back().first->setIsEHScopeEntry();
2040 break;
2041 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2042 // Add the catchpad handlers to the possible destinations. We don't
2043 // continue to the unwind destination of the catchswitch for wasm.
2044 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2045 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
2046 UnwindDests.back().first->setIsEHScopeEntry();
2047 }
2048 break;
2049 } else {
2050 continue;
2051 }
2052 }
2053}
2054
2055/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2056/// many places it could ultimately go. In the IR, we have a single unwind
2057/// destination, but in the machine CFG, we enumerate all the possible blocks.
2058/// This function skips over imaginary basic blocks that hold catchswitch
2059/// instructions, and finds all the "real" machine
2060/// basic block destinations. As those destinations may not be successors of
2061/// EHPadBB, here we also calculate the edge probability to those destinations.
2062/// The passed-in Prob is the edge probability to EHPadBB.
2064 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2065 BranchProbability Prob,
2066 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2067 &UnwindDests) {
2068 EHPersonality Personality =
2070 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2071 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2072 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2073 bool IsSEH = isAsynchronousEHPersonality(Personality);
2074
2075 if (IsWasmCXX) {
2076 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
2077 assert(UnwindDests.size() <= 1 &&
2078 "There should be at most one unwind destination for wasm");
2079 return;
2080 }
2081
2082 while (EHPadBB) {
2083 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2084 BasicBlock *NewEHPadBB = nullptr;
2085 if (isa<LandingPadInst>(Pad)) {
2086 // Stop on landingpads. They are not funclets.
2087 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2088 break;
2089 } else if (isa<CleanupPadInst>(Pad)) {
2090 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2091 // personalities.
2092 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2093 UnwindDests.back().first->setIsEHScopeEntry();
2094 UnwindDests.back().first->setIsEHFuncletEntry();
2095 break;
2096 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2097 // Add the catchpad handlers to the possible destinations.
2098 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2099 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
2100 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2101 if (IsMSVCCXX || IsCoreCLR)
2102 UnwindDests.back().first->setIsEHFuncletEntry();
2103 if (!IsSEH)
2104 UnwindDests.back().first->setIsEHScopeEntry();
2105 }
2106 NewEHPadBB = CatchSwitch->getUnwindDest();
2107 } else {
2108 continue;
2109 }
2110
2111 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2112 if (BPI && NewEHPadBB)
2113 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2114 EHPadBB = NewEHPadBB;
2115 }
2116}
2117
2118void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2119 // Update successor info.
2121 auto UnwindDest = I.getUnwindDest();
2123 BranchProbability UnwindDestProb =
2124 (BPI && UnwindDest)
2125 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2127 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2128 for (auto &UnwindDest : UnwindDests) {
2129 UnwindDest.first->setIsEHPad();
2130 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2131 }
2133
2134 // Create the terminator node.
2135 SDValue Ret =
2137 DAG.setRoot(Ret);
2138}
2139
2140void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2141 report_fatal_error("visitCatchSwitch not yet implemented!");
2142}
2143
2144void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2146 auto &DL = DAG.getDataLayout();
2147 SDValue Chain = getControlRoot();
2150
2151 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2152 // lower
2153 //
2154 // %val = call <ty> @llvm.experimental.deoptimize()
2155 // ret <ty> %val
2156 //
2157 // differently.
2158 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2160 return;
2161 }
2162
2163 if (!FuncInfo.CanLowerReturn) {
2164 unsigned DemoteReg = FuncInfo.DemoteRegister;
2165 const Function *F = I.getParent()->getParent();
2166
2167 // Emit a store of the return value through the virtual register.
2168 // Leave Outs empty so that LowerReturn won't try to load return
2169 // registers the usual way.
2170 SmallVector<EVT, 1> PtrValueVTs;
2171 ComputeValueVTs(TLI, DL,
2172 PointerType::get(F->getContext(),
2174 PtrValueVTs);
2175
2176 SDValue RetPtr =
2177 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2178 SDValue RetOp = getValue(I.getOperand(0));
2179
2180 SmallVector<EVT, 4> ValueVTs, MemVTs;
2182 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2183 &Offsets, 0);
2184 unsigned NumValues = ValueVTs.size();
2185
2186 SmallVector<SDValue, 4> Chains(NumValues);
2187 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2188 for (unsigned i = 0; i != NumValues; ++i) {
2189 // An aggregate return value cannot wrap around the address space, so
2190 // offsets to its parts don't wrap either.
2192 TypeSize::getFixed(Offsets[i]));
2193
2194 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2195 if (MemVTs[i] != ValueVTs[i])
2196 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2197 Chains[i] = DAG.getStore(
2198 Chain, getCurSDLoc(), Val,
2199 // FIXME: better loc info would be nice.
2201 commonAlignment(BaseAlign, Offsets[i]));
2202 }
2203
2205 MVT::Other, Chains);
2206 } else if (I.getNumOperands() != 0) {
2207 SmallVector<EVT, 4> ValueVTs;
2208 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2209 unsigned NumValues = ValueVTs.size();
2210 if (NumValues) {
2211 SDValue RetOp = getValue(I.getOperand(0));
2212
2213 const Function *F = I.getParent()->getParent();
2214
2215 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2216 I.getOperand(0)->getType(), F->getCallingConv(),
2217 /*IsVarArg*/ false, DL);
2218
2219 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2220 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2221 ExtendKind = ISD::SIGN_EXTEND;
2222 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2223 ExtendKind = ISD::ZERO_EXTEND;
2224
2225 LLVMContext &Context = F->getContext();
2226 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2227
2228 for (unsigned j = 0; j != NumValues; ++j) {
2229 EVT VT = ValueVTs[j];
2230
2231 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2232 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2233
2234 CallingConv::ID CC = F->getCallingConv();
2235
2236 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2237 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2238 SmallVector<SDValue, 4> Parts(NumParts);
2240 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2241 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2242
2243 // 'inreg' on function refers to return value
2245 if (RetInReg)
2246 Flags.setInReg();
2247
2248 if (I.getOperand(0)->getType()->isPointerTy()) {
2249 Flags.setPointer();
2250 Flags.setPointerAddrSpace(
2251 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2252 }
2253
2254 if (NeedsRegBlock) {
2255 Flags.setInConsecutiveRegs();
2256 if (j == NumValues - 1)
2257 Flags.setInConsecutiveRegsLast();
2258 }
2259
2260 // Propagate extension type if any
2261 if (ExtendKind == ISD::SIGN_EXTEND)
2262 Flags.setSExt();
2263 else if (ExtendKind == ISD::ZERO_EXTEND)
2264 Flags.setZExt();
2265
2266 for (unsigned i = 0; i < NumParts; ++i) {
2267 Outs.push_back(ISD::OutputArg(Flags,
2268 Parts[i].getValueType().getSimpleVT(),
2269 VT, /*isfixed=*/true, 0, 0));
2270 OutVals.push_back(Parts[i]);
2271 }
2272 }
2273 }
2274 }
2275
2276 // Push in swifterror virtual register as the last element of Outs. This makes
2277 // sure swifterror virtual register will be returned in the swifterror
2278 // physical register.
2279 const Function *F = I.getParent()->getParent();
2280 if (TLI.supportSwiftError() &&
2281 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2282 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2284 Flags.setSwiftError();
2286 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2287 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2288 // Create SDNode for the swifterror virtual register.
2289 OutVals.push_back(
2292 EVT(TLI.getPointerTy(DL))));
2293 }
2294
2295 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2296 CallingConv::ID CallConv =
2299 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2300
2301 // Verify that the target's LowerReturn behaved as expected.
2302 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2303 "LowerReturn didn't return a valid chain!");
2304
2305 // Update the DAG with the new chain value resulting from return lowering.
2306 DAG.setRoot(Chain);
2307}
2308
2309/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2310/// created for it, emit nodes to copy the value into the virtual
2311/// registers.
2313 // Skip empty types
2314 if (V->getType()->isEmptyTy())
2315 return;
2316
2318 if (VMI != FuncInfo.ValueMap.end()) {
2319 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2320 "Unused value assigned virtual registers!");
2321 CopyValueToVirtualRegister(V, VMI->second);
2322 }
2323}
2324
2325/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2326/// the current basic block, add it to ValueMap now so that we'll get a
2327/// CopyTo/FromReg.
2329 // No need to export constants.
2330 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2331
2332 // Already exported?
2333 if (FuncInfo.isExportedInst(V)) return;
2334
2337}
2338
2340 const BasicBlock *FromBB) {
2341 // The operands of the setcc have to be in this block. We don't know
2342 // how to export them from some other block.
2343 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2344 // Can export from current BB.
2345 if (VI->getParent() == FromBB)
2346 return true;
2347
2348 // Is already exported, noop.
2349 return FuncInfo.isExportedInst(V);
2350 }
2351
2352 // If this is an argument, we can export it if the BB is the entry block or
2353 // if it is already exported.
2354 if (isa<Argument>(V)) {
2355 if (FromBB->isEntryBlock())
2356 return true;
2357
2358 // Otherwise, can only export this if it is already exported.
2359 return FuncInfo.isExportedInst(V);
2360 }
2361
2362 // Otherwise, constants can always be exported.
2363 return true;
2364}
2365
2366/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2368SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2369 const MachineBasicBlock *Dst) const {
2371 const BasicBlock *SrcBB = Src->getBasicBlock();
2372 const BasicBlock *DstBB = Dst->getBasicBlock();
2373 if (!BPI) {
2374 // If BPI is not available, set the default probability as 1 / N, where N is
2375 // the number of successors.
2376 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2377 return BranchProbability(1, SuccSize);
2378 }
2379 return BPI->getEdgeProbability(SrcBB, DstBB);
2380}
2381
2382void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2383 MachineBasicBlock *Dst,
2384 BranchProbability Prob) {
2385 if (!FuncInfo.BPI)
2386 Src->addSuccessorWithoutProb(Dst);
2387 else {
2388 if (Prob.isUnknown())
2389 Prob = getEdgeProbability(Src, Dst);
2390 Src->addSuccessor(Dst, Prob);
2391 }
2392}
2393
2394static bool InBlock(const Value *V, const BasicBlock *BB) {
2395 if (const Instruction *I = dyn_cast<Instruction>(V))
2396 return I->getParent() == BB;
2397 return true;
2398}
2399
2400/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2401/// This function emits a branch and is used at the leaves of an OR or an
2402/// AND operator tree.
2403void
2406 MachineBasicBlock *FBB,
2407 MachineBasicBlock *CurBB,
2408 MachineBasicBlock *SwitchBB,
2409 BranchProbability TProb,
2410 BranchProbability FProb,
2411 bool InvertCond) {
2412 const BasicBlock *BB = CurBB->getBasicBlock();
2413
2414 // If the leaf of the tree is a comparison, merge the condition into
2415 // the caseblock.
2416 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2417 // The operands of the cmp have to be in this block. We don't know
2418 // how to export them from some other block. If this is the first block
2419 // of the sequence, no exporting is needed.
2420 if (CurBB == SwitchBB ||
2421 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2422 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2423 ISD::CondCode Condition;
2424 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2425 ICmpInst::Predicate Pred =
2426 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2427 Condition = getICmpCondCode(Pred);
2428 } else {
2429 const FCmpInst *FC = cast<FCmpInst>(Cond);
2430 FCmpInst::Predicate Pred =
2431 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2432 Condition = getFCmpCondCode(Pred);
2433 if (TM.Options.NoNaNsFPMath)
2434 Condition = getFCmpCodeWithoutNaN(Condition);
2435 }
2436
2437 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2438 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2439 SL->SwitchCases.push_back(CB);
2440 return;
2441 }
2442 }
2443
2444 // Create a CaseBlock record representing this branch.
2445 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2447 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2448 SL->SwitchCases.push_back(CB);
2449}
2450
2451// Collect dependencies on V recursively. This is used for the cost analysis in
2452// `shouldKeepJumpConditionsTogether`.
2456 unsigned Depth = 0) {
2457 // Return false if we have an incomplete count.
2459 return false;
2460
2461 auto *I = dyn_cast<Instruction>(V);
2462 if (I == nullptr)
2463 return true;
2464
2465 if (Necessary != nullptr) {
2466 // This instruction is necessary for the other side of the condition so
2467 // don't count it.
2468 if (Necessary->contains(I))
2469 return true;
2470 }
2471
2472 // Already added this dep.
2473 if (!Deps->try_emplace(I, false).second)
2474 return true;
2475
2476 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2477 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2478 Depth + 1))
2479 return false;
2480 return true;
2481}
2482
2484 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,
2485 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2487 if (I.getNumSuccessors() != 2)
2488 return false;
2489
2490 if (!I.isConditional())
2491 return false;
2492
2493 if (Params.BaseCost < 0)
2494 return false;
2495
2496 // Baseline cost.
2497 InstructionCost CostThresh = Params.BaseCost;
2498
2499 BranchProbabilityInfo *BPI = nullptr;
2500 if (Params.LikelyBias || Params.UnlikelyBias)
2501 BPI = FuncInfo.BPI;
2502 if (BPI != nullptr) {
2503 // See if we are either likely to get an early out or compute both lhs/rhs
2504 // of the condition.
2505 BasicBlock *IfFalse = I.getSuccessor(0);
2506 BasicBlock *IfTrue = I.getSuccessor(1);
2507
2508 std::optional<bool> Likely;
2509 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2510 Likely = true;
2511 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2512 Likely = false;
2513
2514 if (Likely) {
2515 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2516 // Its likely we will have to compute both lhs and rhs of condition
2517 CostThresh += Params.LikelyBias;
2518 else {
2519 if (Params.UnlikelyBias < 0)
2520 return false;
2521 // Its likely we will get an early out.
2522 CostThresh -= Params.UnlikelyBias;
2523 }
2524 }
2525 }
2526
2527 if (CostThresh <= 0)
2528 return false;
2529
2530 // Collect "all" instructions that lhs condition is dependent on.
2531 // Use map for stable iteration (to avoid non-determanism of iteration of
2532 // SmallPtrSet). The `bool` value is just a dummy.
2534 collectInstructionDeps(&LhsDeps, Lhs);
2535 // Collect "all" instructions that rhs condition is dependent on AND are
2536 // dependencies of lhs. This gives us an estimate on which instructions we
2537 // stand to save by splitting the condition.
2538 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2539 return false;
2540 // Add the compare instruction itself unless its a dependency on the LHS.
2541 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2542 if (!LhsDeps.contains(RhsI))
2543 RhsDeps.try_emplace(RhsI, false);
2544
2545 const auto &TLI = DAG.getTargetLoweringInfo();
2546 const auto &TTI =
2547 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
2548
2549 InstructionCost CostOfIncluding = 0;
2550 // See if this instruction will need to computed independently of whether RHS
2551 // is.
2552 Value *BrCond = I.getCondition();
2553 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2554 for (const auto *U : Ins->users()) {
2555 // If user is independent of RHS calculation we don't need to count it.
2556 if (auto *UIns = dyn_cast<Instruction>(U))
2557 if (UIns != BrCond && !RhsDeps.contains(UIns))
2558 return false;
2559 }
2560 return true;
2561 };
2562
2563 // Prune instructions from RHS Deps that are dependencies of unrelated
2564 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2565 // arbitrary and just meant to cap the how much time we spend in the pruning
2566 // loop. Its highly unlikely to come into affect.
2567 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2568 // Stop after a certain point. No incorrectness from including too many
2569 // instructions.
2570 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2571 const Instruction *ToDrop = nullptr;
2572 for (const auto &InsPair : RhsDeps) {
2573 if (!ShouldCountInsn(InsPair.first)) {
2574 ToDrop = InsPair.first;
2575 break;
2576 }
2577 }
2578 if (ToDrop == nullptr)
2579 break;
2580 RhsDeps.erase(ToDrop);
2581 }
2582
2583 for (const auto &InsPair : RhsDeps) {
2584 // Finally accumulate latency that we can only attribute to computing the
2585 // RHS condition. Use latency because we are essentially trying to calculate
2586 // the cost of the dependency chain.
2587 // Possible TODO: We could try to estimate ILP and make this more precise.
2588 CostOfIncluding +=
2590
2591 if (CostOfIncluding > CostThresh)
2592 return false;
2593 }
2594 return true;
2595}
2596
2599 MachineBasicBlock *FBB,
2600 MachineBasicBlock *CurBB,
2601 MachineBasicBlock *SwitchBB,
2603 BranchProbability TProb,
2604 BranchProbability FProb,
2605 bool InvertCond) {
2606 // Skip over not part of the tree and remember to invert op and operands at
2607 // next level.
2608 Value *NotCond;
2609 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2610 InBlock(NotCond, CurBB->getBasicBlock())) {
2611 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2612 !InvertCond);
2613 return;
2614 }
2615
2616 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2617 const Value *BOpOp0, *BOpOp1;
2618 // Compute the effective opcode for Cond, taking into account whether it needs
2619 // to be inverted, e.g.
2620 // and (not (or A, B)), C
2621 // gets lowered as
2622 // and (and (not A, not B), C)
2624 if (BOp) {
2625 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2626 ? Instruction::And
2627 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2628 ? Instruction::Or
2630 if (InvertCond) {
2631 if (BOpc == Instruction::And)
2632 BOpc = Instruction::Or;
2633 else if (BOpc == Instruction::Or)
2634 BOpc = Instruction::And;
2635 }
2636 }
2637
2638 // If this node is not part of the or/and tree, emit it as a branch.
2639 // Note that all nodes in the tree should have same opcode.
2640 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2641 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2642 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2643 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2644 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2645 TProb, FProb, InvertCond);
2646 return;
2647 }
2648
2649 // Create TmpBB after CurBB.
2650 MachineFunction::iterator BBI(CurBB);
2653 CurBB->getParent()->insert(++BBI, TmpBB);
2654
2655 if (Opc == Instruction::Or) {
2656 // Codegen X | Y as:
2657 // BB1:
2658 // jmp_if_X TBB
2659 // jmp TmpBB
2660 // TmpBB:
2661 // jmp_if_Y TBB
2662 // jmp FBB
2663 //
2664
2665 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2666 // The requirement is that
2667 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2668 // = TrueProb for original BB.
2669 // Assuming the original probabilities are A and B, one choice is to set
2670 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2671 // A/(1+B) and 2B/(1+B). This choice assumes that
2672 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2673 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2674 // TmpBB, but the math is more complicated.
2675
2676 auto NewTrueProb = TProb / 2;
2677 auto NewFalseProb = TProb / 2 + FProb;
2678 // Emit the LHS condition.
2679 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2680 NewFalseProb, InvertCond);
2681
2682 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2683 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2684 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2685 // Emit the RHS condition into TmpBB.
2686 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2687 Probs[1], InvertCond);
2688 } else {
2689 assert(Opc == Instruction::And && "Unknown merge op!");
2690 // Codegen X & Y as:
2691 // BB1:
2692 // jmp_if_X TmpBB
2693 // jmp FBB
2694 // TmpBB:
2695 // jmp_if_Y TBB
2696 // jmp FBB
2697 //
2698 // This requires creation of TmpBB after CurBB.
2699
2700 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2701 // The requirement is that
2702 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2703 // = FalseProb for original BB.
2704 // Assuming the original probabilities are A and B, one choice is to set
2705 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2706 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2707 // TrueProb for BB1 * FalseProb for TmpBB.
2708
2709 auto NewTrueProb = TProb + FProb / 2;
2710 auto NewFalseProb = FProb / 2;
2711 // Emit the LHS condition.
2712 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2713 NewFalseProb, InvertCond);
2714
2715 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2716 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2717 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2718 // Emit the RHS condition into TmpBB.
2719 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2720 Probs[1], InvertCond);
2721 }
2722}
2723
2724/// If the set of cases should be emitted as a series of branches, return true.
2725/// If we should emit this as a bunch of and/or'd together conditions, return
2726/// false.
2727bool
2728SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2729 if (Cases.size() != 2) return true;
2730
2731 // If this is two comparisons of the same values or'd or and'd together, they
2732 // will get folded into a single comparison, so don't emit two blocks.
2733 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2734 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2735 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2736 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2737 return false;
2738 }
2739
2740 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2741 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2742 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2743 Cases[0].CC == Cases[1].CC &&
2744 isa<Constant>(Cases[0].CmpRHS) &&
2745 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2746 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2747 return false;
2748 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2749 return false;
2750 }
2751
2752 return true;
2753}
2754
2755void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2757
2758 // Update machine-CFG edges.
2759 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2760
2761 if (I.isUnconditional()) {
2762 // Update machine-CFG edges.
2763 BrMBB->addSuccessor(Succ0MBB);
2764
2765 // If this is not a fall-through branch or optimizations are switched off,
2766 // emit the branch.
2767 if (Succ0MBB != NextBlock(BrMBB) ||
2769 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2770 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2771 setValue(&I, Br);
2772 DAG.setRoot(Br);
2773 }
2774
2775 return;
2776 }
2777
2778 // If this condition is one of the special cases we handle, do special stuff
2779 // now.
2780 const Value *CondVal = I.getCondition();
2781 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2782
2783 // If this is a series of conditions that are or'd or and'd together, emit
2784 // this as a sequence of branches instead of setcc's with and/or operations.
2785 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2786 // unpredictable branches, and vector extracts because those jumps are likely
2787 // expensive for any target), this should improve performance.
2788 // For example, instead of something like:
2789 // cmp A, B
2790 // C = seteq
2791 // cmp D, E
2792 // F = setle
2793 // or C, F
2794 // jnz foo
2795 // Emit:
2796 // cmp A, B
2797 // je foo
2798 // cmp D, E
2799 // jle foo
2800 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2801 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2802 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2803 Value *Vec;
2804 const Value *BOp0, *BOp1;
2806 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2807 Opcode = Instruction::And;
2808 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2809 Opcode = Instruction::Or;
2810
2811 if (Opcode &&
2812 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2813 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2815 FuncInfo, I, Opcode, BOp0, BOp1,
2817 Opcode, BOp0, BOp1))) {
2818 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2819 getEdgeProbability(BrMBB, Succ0MBB),
2820 getEdgeProbability(BrMBB, Succ1MBB),
2821 /*InvertCond=*/false);
2822 // If the compares in later blocks need to use values not currently
2823 // exported from this block, export them now. This block should always
2824 // be the first entry.
2825 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2826
2827 // Allow some cases to be rejected.
2828 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2829 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2830 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2831 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2832 }
2833
2834 // Emit the branch for this block.
2835 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2836 SL->SwitchCases.erase(SL->SwitchCases.begin());
2837 return;
2838 }
2839
2840 // Okay, we decided not to do this, remove any inserted MBB's and clear
2841 // SwitchCases.
2842 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2843 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2844
2845 SL->SwitchCases.clear();
2846 }
2847 }
2848
2849 // Create a CaseBlock record representing this branch.
2851 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2852
2853 // Use visitSwitchCase to actually insert the fast branch sequence for this
2854 // cond branch.
2855 visitSwitchCase(CB, BrMBB);
2856}
2857
2858/// visitSwitchCase - Emits the necessary code to represent a single node in
2859/// the binary search tree resulting from lowering a switch instruction.
2861 MachineBasicBlock *SwitchBB) {
2862 SDValue Cond;
2863 SDValue CondLHS = getValue(CB.CmpLHS);
2864 SDLoc dl = CB.DL;
2865
2866 if (CB.CC == ISD::SETTRUE) {
2867 // Branch or fall through to TrueBB.
2868 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2869 SwitchBB->normalizeSuccProbs();
2870 if (CB.TrueBB != NextBlock(SwitchBB)) {
2871 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2872 DAG.getBasicBlock(CB.TrueBB)));
2873 }
2874 return;
2875 }
2876
2877 auto &TLI = DAG.getTargetLoweringInfo();
2878 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2879
2880 // Build the setcc now.
2881 if (!CB.CmpMHS) {
2882 // Fold "(X == true)" to X and "(X == false)" to !X to
2883 // handle common cases produced by branch lowering.
2884 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2885 CB.CC == ISD::SETEQ)
2886 Cond = CondLHS;
2887 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2888 CB.CC == ISD::SETEQ) {
2889 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2890 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2891 } else {
2892 SDValue CondRHS = getValue(CB.CmpRHS);
2893
2894 // If a pointer's DAG type is larger than its memory type then the DAG
2895 // values are zero-extended. This breaks signed comparisons so truncate
2896 // back to the underlying type before doing the compare.
2897 if (CondLHS.getValueType() != MemVT) {
2898 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2899 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2900 }
2901 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2902 }
2903 } else {
2904 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2905
2906 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2907 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2908
2909 SDValue CmpOp = getValue(CB.CmpMHS);
2910 EVT VT = CmpOp.getValueType();
2911
2912 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2913 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2914 ISD::SETLE);
2915 } else {
2916 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2917 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2918 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2919 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2920 }
2921 }
2922
2923 // Update successor info
2924 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2925 // TrueBB and FalseBB are always different unless the incoming IR is
2926 // degenerate. This only happens when running llc on weird IR.
2927 if (CB.TrueBB != CB.FalseBB)
2928 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2929 SwitchBB->normalizeSuccProbs();
2930
2931 // If the lhs block is the next block, invert the condition so that we can
2932 // fall through to the lhs instead of the rhs block.
2933 if (CB.TrueBB == NextBlock(SwitchBB)) {
2934 std::swap(CB.TrueBB, CB.FalseBB);
2935 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2936 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2937 }
2938
2939 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2940 MVT::Other, getControlRoot(), Cond,
2942
2943 setValue(CurInst, BrCond);
2944
2945 // Insert the false branch. Do this even if it's a fall through branch,
2946 // this makes it easier to do DAG optimizations which require inverting
2947 // the branch condition.
2948 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2950
2951 DAG.setRoot(BrCond);
2952}
2953
2954/// visitJumpTable - Emit JumpTable node in the current MBB
2956 // Emit the code for the jump table
2957 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2958 assert(JT.Reg != -1U && "Should lower JT Header first!");
2960 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
2961 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2962 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2963 Index.getValue(1), Table, Index);
2964 DAG.setRoot(BrJumpTable);
2965}
2966
2967/// visitJumpTableHeader - This function emits necessary code to produce index
2968/// in the JumpTable from switch case.
2970 JumpTableHeader &JTH,
2971 MachineBasicBlock *SwitchBB) {
2972 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2973 const SDLoc &dl = *JT.SL;
2974
2975 // Subtract the lowest switch case value from the value being switched on.
2976 SDValue SwitchOp = getValue(JTH.SValue);
2977 EVT VT = SwitchOp.getValueType();
2978 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2979 DAG.getConstant(JTH.First, dl, VT));
2980
2981 // The SDNode we just created, which holds the value being switched on minus
2982 // the smallest case value, needs to be copied to a virtual register so it
2983 // can be used as an index into the jump table in a subsequent basic block.
2984 // This value may be smaller or larger than the target's pointer type, and
2985 // therefore require extension or truncating.
2987 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2988
2989 unsigned JumpTableReg =
2991 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2992 JumpTableReg, SwitchOp);
2993 JT.Reg = JumpTableReg;
2994
2995 if (!JTH.FallthroughUnreachable) {
2996 // Emit the range check for the jump table, and branch to the default block
2997 // for the switch statement if the value being switched on exceeds the
2998 // largest case in the switch.
2999 SDValue CMP = DAG.getSetCC(
3001 Sub.getValueType()),
3002 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3003
3004 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3005 MVT::Other, CopyTo, CMP,
3006 DAG.getBasicBlock(JT.Default));
3007
3008 // Avoid emitting unnecessary branches to the next block.
3009 if (JT.MBB != NextBlock(SwitchBB))
3010 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3011 DAG.getBasicBlock(JT.MBB));
3012
3013 DAG.setRoot(BrCond);
3014 } else {
3015 // Avoid emitting unnecessary branches to the next block.
3016 if (JT.MBB != NextBlock(SwitchBB))
3017 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3018 DAG.getBasicBlock(JT.MBB)));
3019 else
3020 DAG.setRoot(CopyTo);
3021 }
3022}
3023
3024/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3025/// variable if there exists one.
3027 SDValue &Chain) {
3028 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3029 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3030 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3034 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3035 if (Global) {
3036 MachinePointerInfo MPInfo(Global);
3040 MPInfo, Flags, LocationSize::precise(PtrTy.getSizeInBits() / 8),
3041 DAG.getEVTAlign(PtrTy));
3042 DAG.setNodeMemRefs(Node, {MemRef});
3043 }
3044 if (PtrTy != PtrMemTy)
3045 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3046 return SDValue(Node, 0);
3047}
3048
3049/// Codegen a new tail for a stack protector check ParentMBB which has had its
3050/// tail spliced into a stack protector check success bb.
3051///
3052/// For a high level explanation of how this fits into the stack protector
3053/// generation see the comment on the declaration of class
3054/// StackProtectorDescriptor.
3056 MachineBasicBlock *ParentBB) {
3057
3058 // First create the loads to the guard/stack slot for the comparison.
3060 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3061 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3062
3063 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3064 int FI = MFI.getStackProtectorIndex();
3065
3066 SDValue Guard;
3067 SDLoc dl = getCurSDLoc();
3068 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3069 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3070 Align Align =
3071 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
3072
3073 // Generate code to load the content of the guard slot.
3074 SDValue GuardVal = DAG.getLoad(
3075 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3078
3079 if (TLI.useStackGuardXorFP())
3080 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3081
3082 // Retrieve guard check function, nullptr if instrumentation is inlined.
3083 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3084 // The target provides a guard check function to validate the guard value.
3085 // Generate a call to that function with the content of the guard slot as
3086 // argument.
3087 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3088 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3089
3092 Entry.Node = GuardVal;
3093 Entry.Ty = FnTy->getParamType(0);
3094 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3095 Entry.IsInReg = true;
3096 Args.push_back(Entry);
3097
3101 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3102 getValue(GuardCheckFn), std::move(Args));
3103
3104 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3105 DAG.setRoot(Result.second);
3106 return;
3107 }
3108
3109 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3110 // Otherwise, emit a volatile load to retrieve the stack guard value.
3111 SDValue Chain = DAG.getEntryNode();
3112 if (TLI.useLoadStackGuardNode()) {
3113 Guard = getLoadStackGuard(DAG, dl, Chain);
3114 } else {
3115 const Value *IRGuard = TLI.getSDagStackGuard(M);
3116 SDValue GuardPtr = getValue(IRGuard);
3117
3118 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3119 MachinePointerInfo(IRGuard, 0), Align,
3121 }
3122
3123 // Perform the comparison via a getsetcc.
3125 *DAG.getContext(),
3126 Guard.getValueType()),
3127 Guard, GuardVal, ISD::SETNE);
3128
3129 // If the guard/stackslot do not equal, branch to failure MBB.
3130 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3131 MVT::Other, GuardVal.getOperand(0),
3132 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3133 // Otherwise branch to success MBB.
3134 SDValue Br = DAG.getNode(ISD::BR, dl,
3135 MVT::Other, BrCond,
3137
3138 DAG.setRoot(Br);
3139}
3140
3141/// Codegen the failure basic block for a stack protector check.
3142///
3143/// A failure stack protector machine basic block consists simply of a call to
3144/// __stack_chk_fail().
3145///
3146/// For a high level explanation of how this fits into the stack protector
3147/// generation see the comment on the declaration of class
3148/// StackProtectorDescriptor.
3149void
3153 CallOptions.setDiscardResult(true);
3154 SDValue Chain =
3155 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3156 std::nullopt, CallOptions, getCurSDLoc())
3157 .second;
3158 // On PS4/PS5, the "return address" must still be within the calling
3159 // function, even if it's at the very end, so emit an explicit TRAP here.
3160 // Passing 'true' for doesNotReturn above won't generate the trap for us.
3161 if (TM.getTargetTriple().isPS())
3162 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3163 // WebAssembly needs an unreachable instruction after a non-returning call,
3164 // because the function return type can be different from __stack_chk_fail's
3165 // return type (void).
3166 if (TM.getTargetTriple().isWasm())
3167 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3168
3169 DAG.setRoot(Chain);
3170}
3171
3172/// visitBitTestHeader - This function emits necessary code to produce value
3173/// suitable for "bit tests"
3175 MachineBasicBlock *SwitchBB) {
3176 SDLoc dl = getCurSDLoc();
3177
3178 // Subtract the minimum value.
3179 SDValue SwitchOp = getValue(B.SValue);
3180 EVT VT = SwitchOp.getValueType();
3181 SDValue RangeSub =
3182 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3183
3184 // Determine the type of the test operands.
3186 bool UsePtrType = false;
3187 if (!TLI.isTypeLegal(VT)) {
3188 UsePtrType = true;
3189 } else {
3190 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
3191 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
3192 // Switch table case range are encoded into series of masks.
3193 // Just use pointer type, it's guaranteed to fit.
3194 UsePtrType = true;
3195 break;
3196 }
3197 }
3198 SDValue Sub = RangeSub;
3199 if (UsePtrType) {
3200 VT = TLI.getPointerTy(DAG.getDataLayout());
3201 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3202 }
3203
3204 B.RegVT = VT.getSimpleVT();
3205 B.Reg = FuncInfo.CreateReg(B.RegVT);
3206 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3207
3208 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3209
3210 if (!B.FallthroughUnreachable)
3211 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3212 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3213 SwitchBB->normalizeSuccProbs();
3214
3215 SDValue Root = CopyTo;
3216 if (!B.FallthroughUnreachable) {
3217 // Conditional branch to the default block.
3218 SDValue RangeCmp = DAG.getSetCC(dl,
3220 RangeSub.getValueType()),
3221 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3222 ISD::SETUGT);
3223
3224 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3225 DAG.getBasicBlock(B.Default));
3226 }
3227
3228 // Avoid emitting unnecessary branches to the next block.
3229 if (MBB != NextBlock(SwitchBB))
3230 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3231
3232 DAG.setRoot(Root);
3233}
3234
3235/// visitBitTestCase - this function produces one "bit test"
3237 MachineBasicBlock* NextMBB,
3238 BranchProbability BranchProbToNext,
3239 unsigned Reg,
3240 BitTestCase &B,
3241 MachineBasicBlock *SwitchBB) {
3242 SDLoc dl = getCurSDLoc();
3243 MVT VT = BB.RegVT;
3244 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3245 SDValue Cmp;
3246 unsigned PopCount = llvm::popcount(B.Mask);
3248 if (PopCount == 1) {
3249 // Testing for a single bit; just compare the shift count with what it
3250 // would need to be to shift a 1 bit in that position.
3251 Cmp = DAG.getSetCC(
3253 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3254 ISD::SETEQ);
3255 } else if (PopCount == BB.Range) {
3256 // There is only one zero bit in the range, test for it directly.
3257 Cmp = DAG.getSetCC(
3259 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3260 } else {
3261 // Make desired shift
3262 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3263 DAG.getConstant(1, dl, VT), ShiftOp);
3264
3265 // Emit bit tests and jumps
3266 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3267 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3268 Cmp = DAG.getSetCC(
3270 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3271 }
3272
3273 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3274 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3275 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3276 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3277 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3278 // one as they are relative probabilities (and thus work more like weights),
3279 // and hence we need to normalize them to let the sum of them become one.
3280 SwitchBB->normalizeSuccProbs();
3281
3282 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3283 MVT::Other, getControlRoot(),
3284 Cmp, DAG.getBasicBlock(B.TargetBB));
3285
3286 // Avoid emitting unnecessary branches to the next block.
3287 if (NextMBB != NextBlock(SwitchBB))
3288 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3289 DAG.getBasicBlock(NextMBB));
3290
3291 DAG.setRoot(BrAnd);
3292}
3293
3294void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3295 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3296
3297 // Retrieve successors. Look through artificial IR level blocks like
3298 // catchswitch for successors.
3299 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
3300 const BasicBlock *EHPadBB = I.getSuccessor(1);
3301 MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
3302
3303 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3304 // have to do anything here to lower funclet bundles.
3305 assert(!I.hasOperandBundlesOtherThan(
3306 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3307 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3308 LLVMContext::OB_cfguardtarget,
3309 LLVMContext::OB_clang_arc_attachedcall}) &&
3310 "Cannot lower invokes with arbitrary operand bundles yet!");
3311
3312 const Value *Callee(I.getCalledOperand());
3313 const Function *Fn = dyn_cast<Function>(Callee);
3314 if (isa<InlineAsm>(Callee))
3315 visitInlineAsm(I, EHPadBB);
3316 else if (Fn && Fn->isIntrinsic()) {
3317 switch (Fn->getIntrinsicID()) {
3318 default:
3319 llvm_unreachable("Cannot invoke this intrinsic");
3320 case Intrinsic::donothing:
3321 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3322 case Intrinsic::seh_try_begin:
3323 case Intrinsic::seh_scope_begin:
3324 case Intrinsic::seh_try_end:
3325 case Intrinsic::seh_scope_end:
3326 if (EHPadMBB)
3327 // a block referenced by EH table
3328 // so dtor-funclet not removed by opts
3329 EHPadMBB->setMachineBlockAddressTaken();
3330 break;
3331 case Intrinsic::experimental_patchpoint_void:
3332 case Intrinsic::experimental_patchpoint_i64:
3333 visitPatchpoint(I, EHPadBB);
3334 break;
3335 case Intrinsic::experimental_gc_statepoint:
3336 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3337 break;
3338 case Intrinsic::wasm_rethrow: {
3339 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3340 // special because it can be invoked, so we manually lower it to a DAG
3341 // node here.
3343 Ops.push_back(getRoot()); // inchain
3345 Ops.push_back(
3346 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3348 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3350 break;
3351 }
3352 }
3353 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
3354 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3355 // Eventually we will support lowering the @llvm.experimental.deoptimize
3356 // intrinsic, and right now there are no plans to support other intrinsics
3357 // with deopt state.
3358 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3359 } else {
3360 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3361 }
3362
3363 // If the value of the invoke is used outside of its defining block, make it
3364 // available as a virtual register.
3365 // We already took care of the exported value for the statepoint instruction
3366 // during call to the LowerStatepoint.
3367 if (!isa<GCStatepointInst>(I)) {
3369 }
3370
3373 BranchProbability EHPadBBProb =
3374 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3376 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3377
3378 // Update successor info.
3379 addSuccessorWithProb(InvokeMBB, Return);
3380 for (auto &UnwindDest : UnwindDests) {
3381 UnwindDest.first->setIsEHPad();
3382 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3383 }
3384 InvokeMBB->normalizeSuccProbs();
3385
3386 // Drop into normal successor.
3388 DAG.getBasicBlock(Return)));
3389}
3390
3391void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3392 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3393
3394 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3395 // have to do anything here to lower funclet bundles.
3396 assert(!I.hasOperandBundlesOtherThan(
3397 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3398 "Cannot lower callbrs with arbitrary operand bundles yet!");
3399
3400 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3401 visitInlineAsm(I);
3403
3404 // Retrieve successors.
3406 Dests.insert(I.getDefaultDest());
3407 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
3408
3409 // Update successor info.
3410 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3411 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3412 BasicBlock *Dest = I.getIndirectDest(i);
3414 Target->setIsInlineAsmBrIndirectTarget();
3415 Target->setMachineBlockAddressTaken();
3416 Target->setLabelMustBeEmitted();
3417 // Don't add duplicate machine successors.
3418 if (Dests.insert(Dest).second)
3419 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3420 }
3421 CallBrMBB->normalizeSuccProbs();
3422
3423 // Drop into default successor.
3425 MVT::Other, getControlRoot(),
3426 DAG.getBasicBlock(Return)));
3427}
3428
3429void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3430 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3431}
3432
3433void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3435 "Call to landingpad not in landing pad!");
3436
3437 // If there aren't registers to copy the values into (e.g., during SjLj
3438 // exceptions), then don't bother to create these DAG nodes.
3440 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3441 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3442 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3443 return;
3444
3445 // If landingpad's return type is token type, we don't create DAG nodes
3446 // for its exception pointer and selector value. The extraction of exception
3447 // pointer or selector value from token type landingpads is not currently
3448 // supported.
3449 if (LP.getType()->isTokenTy())
3450 return;
3451
3452 SmallVector<EVT, 2> ValueVTs;
3453 SDLoc dl = getCurSDLoc();
3454 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3455 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3456
3457 // Get the two live-in registers as SDValues. The physregs have already been
3458 // copied into virtual registers.
3459 SDValue Ops[2];
3461 Ops[0] = DAG.getZExtOrTrunc(
3465 dl, ValueVTs[0]);
3466 } else {
3467 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3468 }
3469 Ops[1] = DAG.getZExtOrTrunc(
3473 dl, ValueVTs[1]);
3474
3475 // Merge into one.
3477 DAG.getVTList(ValueVTs), Ops);
3478 setValue(&LP, Res);
3479}
3480
3483 // Update JTCases.
3484 for (JumpTableBlock &JTB : SL->JTCases)
3485 if (JTB.first.HeaderBB == First)
3486 JTB.first.HeaderBB = Last;
3487
3488 // Update BitTestCases.
3489 for (BitTestBlock &BTB : SL->BitTestCases)
3490 if (BTB.Parent == First)
3491 BTB.Parent = Last;
3492}
3493
3494void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3495 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3496
3497 // Update machine-CFG edges with unique successors.
3499 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3500 BasicBlock *BB = I.getSuccessor(i);
3501 bool Inserted = Done.insert(BB).second;
3502 if (!Inserted)
3503 continue;
3504
3505 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3506 addSuccessorWithProb(IndirectBrMBB, Succ);
3507 }
3508 IndirectBrMBB->normalizeSuccProbs();
3509
3511 MVT::Other, getControlRoot(),
3512 getValue(I.getAddress())));
3513}
3514
3515void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3517 return;
3518
3519 // We may be able to ignore unreachable behind a noreturn call.
3521 if (const CallInst *Call = dyn_cast_or_null<CallInst>(I.getPrevNode())) {
3522 if (Call->doesNotReturn())
3523 return;
3524 }
3525 }
3526
3527 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3528}
3529
3530void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3532 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3533 Flags.copyFMF(*FPOp);
3534
3535 SDValue Op = getValue(I.getOperand(0));
3536 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3537 Op, Flags);
3538 setValue(&I, UnNodeValue);
3539}
3540
3541void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3543 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3544 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3545 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3546 }
3547 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3548 Flags.setExact(ExactOp->isExact());
3549 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3550 Flags.setDisjoint(DisjointOp->isDisjoint());
3551 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3552 Flags.copyFMF(*FPOp);
3553
3554 SDValue Op1 = getValue(I.getOperand(0));
3555 SDValue Op2 = getValue(I.getOperand(1));
3556 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3557 Op1, Op2, Flags);
3558 setValue(&I, BinNodeValue);
3559}
3560
3561void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3562 SDValue Op1 = getValue(I.getOperand(0));
3563 SDValue Op2 = getValue(I.getOperand(1));
3564
3566 Op1.getValueType(), DAG.getDataLayout());
3567
3568 // Coerce the shift amount to the right type if we can. This exposes the
3569 // truncate or zext to optimization early.
3570 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3572 "Unexpected shift type");
3573 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3574 }
3575
3576 bool nuw = false;
3577 bool nsw = false;
3578 bool exact = false;
3579
3580 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3581
3582 if (const OverflowingBinaryOperator *OFBinOp =
3583 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3584 nuw = OFBinOp->hasNoUnsignedWrap();
3585 nsw = OFBinOp->hasNoSignedWrap();
3586 }
3587 if (const PossiblyExactOperator *ExactOp =
3588 dyn_cast<const PossiblyExactOperator>(&I))
3589 exact = ExactOp->isExact();
3590 }
3592 Flags.setExact(exact);
3593 Flags.setNoSignedWrap(nsw);
3594 Flags.setNoUnsignedWrap(nuw);
3595 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3596 Flags);
3597 setValue(&I, Res);
3598}
3599
3600void SelectionDAGBuilder::visitSDiv(const User &I) {
3601 SDValue Op1 = getValue(I.getOperand(0));
3602 SDValue Op2 = getValue(I.getOperand(1));
3603
3605 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3606 cast<PossiblyExactOperator>(&I)->isExact());
3608 Op2, Flags));
3609}
3610
3611void SelectionDAGBuilder::visitICmp(const User &I) {
3613 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3614 predicate = IC->getPredicate();
3615 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3616 predicate = ICmpInst::Predicate(IC->getPredicate());
3617 SDValue Op1 = getValue(I.getOperand(0));
3618 SDValue Op2 = getValue(I.getOperand(1));
3619 ISD::CondCode Opcode = getICmpCondCode(predicate);
3620
3621 auto &TLI = DAG.getTargetLoweringInfo();
3622 EVT MemVT =
3623 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3624
3625 // If a pointer's DAG type is larger than its memory type then the DAG values
3626 // are zero-extended. This breaks signed comparisons so truncate back to the
3627 // underlying type before doing the compare.
3628 if (Op1.getValueType() != MemVT) {
3629 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3630 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3631 }
3632
3634 I.getType());
3635 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3636}
3637
3638void SelectionDAGBuilder::visitFCmp(const User &I) {
3640 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3641 predicate = FC->getPredicate();
3642 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3643 predicate = FCmpInst::Predicate(FC->getPredicate());
3644 SDValue Op1 = getValue(I.getOperand(0));
3645 SDValue Op2 = getValue(I.getOperand(1));
3646
3647 ISD::CondCode Condition = getFCmpCondCode(predicate);
3648 auto *FPMO = cast<FPMathOperator>(&I);
3649 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3650 Condition = getFCmpCodeWithoutNaN(Condition);
3651
3653 Flags.copyFMF(*FPMO);
3654 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3655
3657 I.getType());
3658 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3659}
3660
3661// Check if the condition of the select has one use or two users that are both
3662// selects with the same condition.
3663static bool hasOnlySelectUsers(const Value *Cond) {
3664 return llvm::all_of(Cond->users(), [](const Value *V) {
3665 return isa<SelectInst>(V);
3666 });
3667}
3668
3669void SelectionDAGBuilder::visitSelect(const User &I) {
3670 SmallVector<EVT, 4> ValueVTs;
3672 ValueVTs);
3673 unsigned NumValues = ValueVTs.size();
3674 if (NumValues == 0) return;
3675
3676 SmallVector<SDValue, 4> Values(NumValues);
3677 SDValue Cond = getValue(I.getOperand(0));
3678 SDValue LHSVal = getValue(I.getOperand(1));
3679 SDValue RHSVal = getValue(I.getOperand(2));
3680 SmallVector<SDValue, 1> BaseOps(1, Cond);
3681 ISD::NodeType OpCode =
3682 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3683
3684 bool IsUnaryAbs = false;
3685 bool Negate = false;
3686
3688 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3689 Flags.copyFMF(*FPOp);
3690
3691 Flags.setUnpredictable(
3692 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3693
3694 // Min/max matching is only viable if all output VTs are the same.
3695 if (all_equal(ValueVTs)) {
3696 EVT VT = ValueVTs[0];
3697 LLVMContext &Ctx = *DAG.getContext();
3698 auto &TLI = DAG.getTargetLoweringInfo();
3699
3700 // We care about the legality of the operation after it has been type
3701 // legalized.
3702 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3703 VT = TLI.getTypeToTransformTo(Ctx, VT);
3704
3705 // If the vselect is legal, assume we want to leave this as a vector setcc +
3706 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3707 // min/max is legal on the scalar type.
3708 bool UseScalarMinMax = VT.isVector() &&
3710
3711 // ValueTracking's select pattern matching does not account for -0.0,
3712 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3713 // -0.0 is less than +0.0.
3714 Value *LHS, *RHS;
3715 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3717 switch (SPR.Flavor) {
3718 case SPF_UMAX: Opc = ISD::UMAX; break;
3719 case SPF_UMIN: Opc = ISD::UMIN; break;
3720 case SPF_SMAX: Opc = ISD::SMAX; break;
3721 case SPF_SMIN: Opc = ISD::SMIN; break;
3722 case SPF_FMINNUM:
3723 switch (SPR.NaNBehavior) {
3724 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3725 case SPNB_RETURNS_NAN: break;
3726 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3727 case SPNB_RETURNS_ANY:
3729 (UseScalarMinMax &&
3731 Opc = ISD::FMINNUM;
3732 break;
3733 }
3734 break;
3735 case SPF_FMAXNUM:
3736 switch (SPR.NaNBehavior) {
3737 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3738 case SPNB_RETURNS_NAN: break;
3739 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3740 case SPNB_RETURNS_ANY:
3742 (UseScalarMinMax &&
3744 Opc = ISD::FMAXNUM;
3745 break;
3746 }
3747 break;
3748 case SPF_NABS:
3749 Negate = true;
3750 [[fallthrough]];
3751 case SPF_ABS:
3752 IsUnaryAbs = true;
3753 Opc = ISD::ABS;
3754 break;
3755 default: break;
3756 }
3757
3758 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3759 (TLI.isOperationLegalOrCustomOrPromote(Opc, VT) ||
3760 (UseScalarMinMax &&
3761 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3762 // If the underlying comparison instruction is used by any other
3763 // instruction, the consumed instructions won't be destroyed, so it is
3764 // not profitable to convert to a min/max.
3765 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3766 OpCode = Opc;
3767 LHSVal = getValue(LHS);
3768 RHSVal = getValue(RHS);
3769 BaseOps.clear();
3770 }
3771
3772 if (IsUnaryAbs) {
3773 OpCode = Opc;
3774 LHSVal = getValue(LHS);
3775 BaseOps.clear();
3776 }
3777 }
3778
3779 if (IsUnaryAbs) {
3780 for (unsigned i = 0; i != NumValues; ++i) {
3781 SDLoc dl = getCurSDLoc();
3782 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3783 Values[i] =
3784 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3785 if (Negate)
3786 Values[i] = DAG.getNegative(Values[i], dl, VT);
3787 }
3788 } else {
3789 for (unsigned i = 0; i != NumValues; ++i) {
3790 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3791 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3792 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3793 Values[i] = DAG.getNode(
3794 OpCode, getCurSDLoc(),
3795 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3796 }
3797 }
3798
3800 DAG.getVTList(ValueVTs), Values));
3801}
3802
3803void SelectionDAGBuilder::visitTrunc(const User &I) {
3804 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3805 SDValue N = getValue(I.getOperand(0));
3807 I.getType());
3809}
3810
3811void SelectionDAGBuilder::visitZExt(const User &I) {
3812 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3813 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3814 SDValue N = getValue(I.getOperand(0));
3815 auto &TLI = DAG.getTargetLoweringInfo();
3816 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3817
3819 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3820 Flags.setNonNeg(PNI->hasNonNeg());
3821
3822 // Eagerly use nonneg information to canonicalize towards sign_extend if
3823 // that is the target's preference.
3824 // TODO: Let the target do this later.
3825 if (Flags.hasNonNeg() &&
3826 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
3828 return;
3829 }
3830
3831 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
3832}
3833
3834void SelectionDAGBuilder::visitSExt(const User &I) {
3835 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3836 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3837 SDValue N = getValue(I.getOperand(0));
3839 I.getType());
3841}
3842
3843void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3844 // FPTrunc is never a no-op cast, no need to check
3845 SDValue N = getValue(I.getOperand(0));
3846 SDLoc dl = getCurSDLoc();
3848 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3849 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3851 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3852}
3853
3854void SelectionDAGBuilder::visitFPExt(const User &I) {
3855 // FPExt is never a no-op cast, no need to check
3856 SDValue N = getValue(I.getOperand(0));
3858 I.getType());
3860}
3861
3862void SelectionDAGBuilder::visitFPToUI(const User &I) {
3863 // FPToUI is never a no-op cast, no need to check
3864 SDValue N = getValue(I.getOperand(0));
3866 I.getType());
3868}
3869
3870void SelectionDAGBuilder::visitFPToSI(const User &I) {
3871 // FPToSI is never a no-op cast, no need to check
3872 SDValue N = getValue(I.getOperand(0));
3874 I.getType());
3876}
3877
3878void SelectionDAGBuilder::visitUIToFP(const User &I) {
3879 // UIToFP is never a no-op cast, no need to check
3880 SDValue N = getValue(I.getOperand(0));
3882 I.getType());
3884}
3885
3886void SelectionDAGBuilder::visitSIToFP(const User &I) {
3887 // SIToFP is never a no-op cast, no need to check
3888 SDValue N = getValue(I.getOperand(0));
3890 I.getType());
3892}
3893
3894void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3895 // What to do depends on the size of the integer and the size of the pointer.
3896 // We can either truncate, zero extend, or no-op, accordingly.
3897 SDValue N = getValue(I.getOperand(0));
3898 auto &TLI = DAG.getTargetLoweringInfo();
3900 I.getType());
3901 EVT PtrMemVT =
3902 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3903 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3904 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3905 setValue(&I, N);
3906}
3907
3908void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3909 // What to do depends on the size of the integer and the size of the pointer.
3910 // We can either truncate, zero extend, or no-op, accordingly.
3911 SDValue N = getValue(I.getOperand(0));
3912 auto &TLI = DAG.getTargetLoweringInfo();
3913 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3914 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3915 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3916 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3917 setValue(&I, N);
3918}
3919
3920void SelectionDAGBuilder::visitBitCast(const User &I) {
3921 SDValue N = getValue(I.getOperand(0));
3922 SDLoc dl = getCurSDLoc();
3924 I.getType());
3925
3926 // BitCast assures us that source and destination are the same size so this is
3927 // either a BITCAST or a no-op.
3928 if (DestVT != N.getValueType())
3930 DestVT, N)); // convert types.
3931 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3932 // might fold any kind of constant expression to an integer constant and that
3933 // is not what we are looking for. Only recognize a bitcast of a genuine
3934 // constant integer as an opaque constant.
3935 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3936 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3937 /*isOpaque*/true));
3938 else
3939 setValue(&I, N); // noop cast.
3940}
3941
3942void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3944 const Value *SV = I.getOperand(0);
3945 SDValue N = getValue(SV);
3946 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3947
3948 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3949 unsigned DestAS = I.getType()->getPointerAddressSpace();
3950
3951 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3952 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3953
3954 setValue(&I, N);
3955}
3956
3957void SelectionDAGBuilder::visitInsertElement(const User &I) {
3959 SDValue InVec = getValue(I.getOperand(0));
3960 SDValue InVal = getValue(I.getOperand(1));
3961 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3964 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3965 InVec, InVal, InIdx));
3966}
3967
3968void SelectionDAGBuilder::visitExtractElement(const User &I) {
3970 SDValue InVec = getValue(I.getOperand(0));
3971 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3974 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3975 InVec, InIdx));
3976}
3977
3978void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3979 SDValue Src1 = getValue(I.getOperand(0));
3980 SDValue Src2 = getValue(I.getOperand(1));
3982 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3983 Mask = SVI->getShuffleMask();
3984 else
3985 Mask = cast<ConstantExpr>(I).getShuffleMask();
3986 SDLoc DL = getCurSDLoc();
3988 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3989 EVT SrcVT = Src1.getValueType();
3990
3991 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3992 VT.isScalableVector()) {
3993 // Canonical splat form of first element of first input vector.
3994 SDValue FirstElt =
3997 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3998 return;
3999 }
4000
4001 // For now, we only handle splats for scalable vectors.
4002 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4003 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4004 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4005
4006 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4007 unsigned MaskNumElts = Mask.size();
4008
4009 if (SrcNumElts == MaskNumElts) {
4010 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4011 return;
4012 }
4013
4014 // Normalize the shuffle vector since mask and vector length don't match.
4015 if (SrcNumElts < MaskNumElts) {
4016 // Mask is longer than the source vectors. We can use concatenate vector to
4017 // make the mask and vectors lengths match.
4018
4019 if (MaskNumElts % SrcNumElts == 0) {
4020 // Mask length is a multiple of the source vector length.
4021 // Check if the shuffle is some kind of concatenation of the input
4022 // vectors.
4023 unsigned NumConcat = MaskNumElts / SrcNumElts;
4024 bool IsConcat = true;
4025 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4026 for (unsigned i = 0; i != MaskNumElts; ++i) {
4027 int Idx = Mask[i];
4028 if (Idx < 0)
4029 continue;
4030 // Ensure the indices in each SrcVT sized piece are sequential and that
4031 // the same source is used for the whole piece.
4032 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4033 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4034 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4035 IsConcat = false;
4036 break;
4037 }
4038 // Remember which source this index came from.
4039 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4040 }
4041
4042 // The shuffle is concatenating multiple vectors together. Just emit
4043 // a CONCAT_VECTORS operation.
4044 if (IsConcat) {
4045 SmallVector<SDValue, 8> ConcatOps;
4046 for (auto Src : ConcatSrcs) {
4047 if (Src < 0)
4048 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4049 else if (Src == 0)
4050 ConcatOps.push_back(Src1);
4051 else
4052 ConcatOps.push_back(Src2);
4053 }
4054 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4055 return;
4056 }
4057 }
4058
4059 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4060 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4061 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4062 PaddedMaskNumElts);
4063
4064 // Pad both vectors with undefs to make them the same length as the mask.
4065 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4066
4067 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4068 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4069 MOps1[0] = Src1;
4070 MOps2[0] = Src2;
4071
4072 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4073 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4074
4075 // Readjust mask for new input vector length.
4076 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4077 for (unsigned i = 0; i != MaskNumElts; ++i) {
4078 int Idx = Mask[i];
4079 if (Idx >= (int)SrcNumElts)
4080 Idx -= SrcNumElts - PaddedMaskNumElts;
4081 MappedOps[i] = Idx;
4082 }
4083
4084 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4085
4086 // If the concatenated vector was padded, extract a subvector with the
4087 // correct number of elements.
4088 if (MaskNumElts != PaddedMaskNumElts)
4091
4092 setValue(&I, Result);
4093 return;
4094 }
4095
4096 if (SrcNumElts > MaskNumElts) {
4097 // Analyze the access pattern of the vector to see if we can extract
4098 // two subvectors and do the shuffle.
4099 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
4100 bool CanExtract = true;
4101 for (int Idx : Mask) {
4102 unsigned Input = 0;
4103 if (Idx < 0)
4104 continue;
4105
4106 if (Idx >= (int)SrcNumElts) {
4107 Input = 1;
4108 Idx -= SrcNumElts;
4109 }
4110
4111 // If all the indices come from the same MaskNumElts sized portion of
4112 // the sources we can use extract. Also make sure the extract wouldn't
4113 // extract past the end of the source.
4114 int NewStartIdx = alignDown(Idx, MaskNumElts);
4115 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4116 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4117 CanExtract = false;
4118 // Make sure we always update StartIdx as we use it to track if all
4119 // elements are undef.
4120 StartIdx[Input] = NewStartIdx;
4121 }
4122
4123 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4124 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4125 return;
4126 }
4127 if (CanExtract) {
4128 // Extract appropriate subvector and generate a vector shuffle
4129 for (unsigned Input = 0; Input < 2; ++Input) {
4130 SDValue &Src = Input == 0 ? Src1 : Src2;
4131 if (StartIdx[Input] < 0)
4132 Src = DAG.getUNDEF(VT);
4133 else {
4134 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4135 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4136 }
4137 }
4138
4139 // Calculate new mask.
4140 SmallVector<int, 8> MappedOps(Mask);
4141 for (int &Idx : MappedOps) {
4142 if (Idx >= (int)SrcNumElts)
4143 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4144 else if (Idx >= 0)
4145 Idx -= StartIdx[0];
4146 }
4147
4148 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4149 return;
4150 }
4151 }
4152
4153 // We can't use either concat vectors or extract subvectors so fall back to
4154 // replacing the shuffle with extract and build vector.
4155 // to insert and build vector.
4156 EVT EltVT = VT.getVectorElementType();
4158 for (int Idx : Mask) {
4159 SDValue Res;
4160
4161 if (Idx < 0) {
4162 Res = DAG.getUNDEF(EltVT);
4163 } else {
4164 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4165 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4166
4167 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4169 }
4170
4171 Ops.push_back(Res);
4172 }
4173
4174 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4175}
4176
4177void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4178 ArrayRef<unsigned> Indices = I.getIndices();
4179 const Value *Op0 = I.getOperand(0);
4180 const Value *Op1 = I.getOperand(1);
4181 Type *AggTy = I.getType();
4182 Type *ValTy = Op1->getType();
4183 bool IntoUndef = isa<UndefValue>(Op0);
4184 bool FromUndef = isa<UndefValue>(Op1);
4185
4186 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4187
4189 SmallVector<EVT, 4> AggValueVTs;
4190 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4191 SmallVector<EVT, 4> ValValueVTs;
4192 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4193
4194 unsigned NumAggValues = AggValueVTs.size();
4195 unsigned NumValValues = ValValueVTs.size();
4196 SmallVector<SDValue, 4> Values(NumAggValues);
4197
4198 // Ignore an insertvalue that produces an empty object
4199 if (!NumAggValues) {
4200 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4201 return;
4202 }
4203
4204 SDValue Agg = getValue(Op0);
4205 unsigned i = 0;
4206 // Copy the beginning value(s) from the original aggregate.
4207 for (; i != LinearIndex; ++i)
4208 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4209 SDValue(Agg.getNode(), Agg.getResNo() + i);
4210 // Copy values from the inserted value(s).
4211 if (NumValValues) {
4212 SDValue Val = getValue(Op1);
4213 for (; i != LinearIndex + NumValValues; ++i)
4214 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4215 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4216 }
4217 // Copy remaining value(s) from the original aggregate.
4218 for (; i != NumAggValues; ++i)
4219 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4220 SDValue(Agg.getNode(), Agg.getResNo() + i);
4221
4223 DAG.getVTList(AggValueVTs), Values));
4224}
4225
4226void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4227 ArrayRef<unsigned> Indices = I.getIndices();
4228 const Value *Op0 = I.getOperand(0);
4229 Type *AggTy = Op0->getType();
4230 Type *ValTy = I.getType();
4231 bool OutOfUndef = isa<UndefValue>(Op0);
4232
4233 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4234
4236 SmallVector<EVT, 4> ValValueVTs;
4237 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4238
4239 unsigned NumValValues = ValValueVTs.size();
4240
4241 // Ignore a extractvalue that produces an empty object
4242 if (!NumValValues) {
4243 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4244 return;
4245 }
4246
4247 SmallVector<SDValue, 4> Values(NumValValues);
4248
4249 SDValue Agg = getValue(Op0);
4250 // Copy out the selected value(s).
4251 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4252 Values[i - LinearIndex] =
4253 OutOfUndef ?
4254 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4255 SDValue(Agg.getNode(), Agg.getResNo() + i);
4256
4258 DAG.getVTList(ValValueVTs), Values));
4259}
4260
4261void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4262 Value *Op0 = I.getOperand(0);
4263 // Note that the pointer operand may be a vector of pointers. Take the scalar
4264 // element which holds a pointer.
4265 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4266 SDValue N = getValue(Op0);
4267 SDLoc dl = getCurSDLoc();
4268 auto &TLI = DAG.getTargetLoweringInfo();
4269
4270 // Normalize Vector GEP - all scalar operands should be converted to the
4271 // splat vector.
4272 bool IsVectorGEP = I.getType()->isVectorTy();
4273 ElementCount VectorElementCount =
4274 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4276
4277 if (IsVectorGEP && !N.getValueType().isVector()) {
4279 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
4280 N = DAG.getSplat(VT, dl, N);
4281 }
4282
4284 GTI != E; ++GTI) {
4285 const Value *Idx = GTI.getOperand();
4286 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4287 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4288 if (Field) {
4289 // N = N + Offset
4292
4293 // In an inbounds GEP with an offset that is nonnegative even when
4294 // interpreted as signed, assume there is no unsigned overflow.
4296 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
4297 Flags.setNoUnsignedWrap(true);
4298
4299 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
4300 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
4301 }
4302 } else {
4303 // IdxSize is the width of the arithmetic according to IR semantics.
4304 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4305 // (and fix up the result later).
4306 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4307 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4308 TypeSize ElementSize =
4309 GTI.getSequentialElementStride(DAG.getDataLayout());
4310 // We intentionally mask away the high bits here; ElementSize may not
4311 // fit in IdxTy.
4312 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
4313 bool ElementScalable = ElementSize.isScalable();
4314
4315 // If this is a scalar constant or a splat vector of constants,
4316 // handle it quickly.
4317 const auto *C = dyn_cast<Constant>(Idx);
4318 if (C && isa<VectorType>(C->getType()))
4319 C = C->getSplatValue();
4320
4321 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4322 if (CI && CI->isZero())
4323 continue;
4324 if (CI && !ElementScalable) {
4325 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4327 SDValue OffsVal;
4328 if (IsVectorGEP)
4329 OffsVal = DAG.getConstant(
4330 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4331 else
4332 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4333
4334 // In an inbounds GEP with an offset that is nonnegative even when
4335 // interpreted as signed, assume there is no unsigned overflow.
4337 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
4338 Flags.setNoUnsignedWrap(true);
4339
4340 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4341
4342 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
4343 continue;
4344 }
4345
4346 // N = N + Idx * ElementMul;
4347 SDValue IdxN = getValue(Idx);
4348
4349 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4351 VectorElementCount);
4352 IdxN = DAG.getSplat(VT, dl, IdxN);
4353 }
4354
4355 // If the index is smaller or larger than intptr_t, truncate or extend
4356 // it.
4357 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4358
4359 if (ElementScalable) {
4360 EVT VScaleTy = N.getValueType().getScalarType();
4361 SDValue VScale = DAG.getNode(
4362 ISD::VSCALE, dl, VScaleTy,
4363 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4364 if (IsVectorGEP)
4365 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4366 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
4367 } else {
4368 // If this is a multiply by a power of two, turn it into a shl
4369 // immediately. This is a very common case.
4370 if (ElementMul != 1) {
4371 if (ElementMul.isPowerOf2()) {
4372 unsigned Amt = ElementMul.logBase2();
4373 IdxN = DAG.getNode(ISD::SHL, dl,
4374 N.getValueType(), IdxN,
4375 DAG.getConstant(Amt, dl, IdxN.getValueType()));
4376 } else {
4377 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4378 IdxN.getValueType());
4379 IdxN = DAG.getNode(ISD::MUL, dl,
4380 N.getValueType(), IdxN, Scale);
4381 }
4382 }
4383 }
4384
4385 N = DAG.getNode(ISD::ADD, dl,
4386 N.getValueType(), N, IdxN);
4387 }
4388 }
4389
4390 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4391 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4392 if (IsVectorGEP) {
4393 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4394 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4395 }
4396
4397 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4398 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4399
4400 setValue(&I, N);
4401}
4402
4403void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4404 // If this is a fixed sized alloca in the entry block of the function,
4405 // allocate it statically on the stack.
4406 if (FuncInfo.StaticAllocaMap.count(&I))
4407 return; // getValue will auto-populate this.
4408
4409 SDLoc dl = getCurSDLoc();
4410 Type *Ty = I.getAllocatedType();
4412 auto &DL = DAG.getDataLayout();
4413 TypeSize TySize = DL.getTypeAllocSize(Ty);
4414 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4415
4416 SDValue AllocSize = getValue(I.getArraySize());
4417
4418 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4419 if (AllocSize.getValueType() != IntPtr)
4420 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4421
4422 if (TySize.isScalable())
4423 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4424 DAG.getVScale(dl, IntPtr,
4425 APInt(IntPtr.getScalarSizeInBits(),
4426 TySize.getKnownMinValue())));
4427 else {
4428 SDValue TySizeValue =
4430 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4431 DAG.getZExtOrTrunc(TySizeValue, dl, IntPtr));
4432 }
4433
4434 // Handle alignment. If the requested alignment is less than or equal to
4435 // the stack alignment, ignore it. If the size is greater than or equal to
4436 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4438 if (*Alignment <= StackAlign)
4439 Alignment = std::nullopt;
4440
4441 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4442 // Round the size of the allocation up to the stack alignment size
4443 // by add SA-1 to the size. This doesn't overflow because we're computing
4444 // an address inside an alloca.
4446 Flags.setNoUnsignedWrap(true);
4447 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4448 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4449
4450 // Mask out the low bits for alignment purposes.
4451 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4452 DAG.getConstant(~StackAlignMask, dl, IntPtr));
4453
4454 SDValue Ops[] = {
4455 getRoot(), AllocSize,
4456 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4457 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4458 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4459 setValue(&I, DSA);
4460 DAG.setRoot(DSA.getValue(1));
4461
4463}
4464
4465static const MDNode *getRangeMetadata(const Instruction &I) {
4466 // If !noundef is not present, then !range violation results in a poison
4467 // value rather than immediate undefined behavior. In theory, transferring
4468 // these annotations to SDAG is fine, but in practice there are key SDAG
4469 // transforms that are known not to be poison-safe, such as folding logical
4470 // and/or to bitwise and/or. For now, only transfer !range if !noundef is
4471 // also present.
4472 if (!I.hasMetadata(LLVMContext::MD_noundef))
4473 return nullptr;
4474 return I.getMetadata(LLVMContext::MD_range);
4475}
4476
4477void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4478 if (I.isAtomic())
4479 return visitAtomicLoad(I);
4480
4482 const Value *SV = I.getOperand(0);
4483 if (TLI.supportSwiftError()) {
4484 // Swifterror values can come from either a function parameter with
4485 // swifterror attribute or an alloca with swifterror attribute.
4486 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4487 if (Arg->hasSwiftErrorAttr())
4488 return visitLoadFromSwiftError(I);
4489 }
4490
4491 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4492 if (Alloca->isSwiftError())
4493 return visitLoadFromSwiftError(I);
4494 }
4495 }
4496
4497 SDValue Ptr = getValue(SV);
4498
4499 Type *Ty = I.getType();
4500 SmallVector<EVT, 4> ValueVTs, MemVTs;
4502 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4503 unsigned NumValues = ValueVTs.size();
4504 if (NumValues == 0)
4505 return;
4506
4507 Align Alignment = I.getAlign();
4508 AAMDNodes AAInfo = I.getAAMetadata();
4509 const MDNode *Ranges = getRangeMetadata(I);
4510 bool isVolatile = I.isVolatile();
4511 MachineMemOperand::Flags MMOFlags =
4513
4514 SDValue Root;
4515 bool ConstantMemory = false;
4516 if (isVolatile)
4517 // Serialize volatile loads with other side effects.
4518 Root = getRoot();
4519 else if (NumValues > MaxParallelChains)
4520 Root = getMemoryRoot();
4521 else if (AA &&
4523 SV,
4525 AAInfo))) {
4526 // Do not serialize (non-volatile) loads of constant memory with anything.
4527 Root = DAG.getEntryNode();
4528 ConstantMemory = true;
4530 } else {
4531 // Do not serialize non-volatile loads against each other.
4532 Root = DAG.getRoot();
4533 }
4534
4535 SDLoc dl = getCurSDLoc();
4536
4537 if (isVolatile)
4538 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4539
4540 SmallVector<SDValue, 4> Values(NumValues);
4541 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4542
4543 unsigned ChainI = 0;
4544 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4545 // Serializing loads here may result in excessive register pressure, and
4546 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4547 // could recover a bit by hoisting nodes upward in the chain by recognizing
4548 // they are side-effect free or do not alias. The optimizer should really
4549 // avoid this case by converting large object/array copies to llvm.memcpy
4550 // (MaxParallelChains should always remain as failsafe).
4551 if (ChainI == MaxParallelChains) {
4552 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4553 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4554 ArrayRef(Chains.data(), ChainI));
4555 Root = Chain;
4556 ChainI = 0;
4557 }
4558
4559 // TODO: MachinePointerInfo only supports a fixed length offset.
4560 MachinePointerInfo PtrInfo =
4561 !Offsets[i].isScalable() || Offsets[i].isZero()
4562 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4564
4565 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4566 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
4567 MMOFlags, AAInfo, Ranges);
4568 Chains[ChainI] = L.getValue(1);
4569
4570 if (MemVTs[i] != ValueVTs[i])
4571 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4572
4573 Values[i] = L;
4574 }
4575
4576 if (!ConstantMemory) {
4577 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4578 ArrayRef(Chains.data(), ChainI));
4579 if (isVolatile)
4580 DAG.setRoot(Chain);
4581 else
4582 PendingLoads.push_back(Chain);
4583 }
4584
4586 DAG.getVTList(ValueVTs), Values));
4587}
4588
4589void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4591 "call visitStoreToSwiftError when backend supports swifterror");
4592
4593 SmallVector<EVT, 4> ValueVTs;
4595 const Value *SrcV = I.getOperand(0);
4597 SrcV->getType(), ValueVTs, &Offsets, 0);
4598 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4599 "expect a single EVT for swifterror");
4600
4601 SDValue Src = getValue(SrcV);
4602 // Create a virtual register, then update the virtual register.
4603 Register VReg =
4604 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4605 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4606 // Chain can be getRoot or getControlRoot.
4607 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4608 SDValue(Src.getNode(), Src.getResNo()));
4609 DAG.setRoot(CopyNode);
4610}
4611
4612void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4614 "call visitLoadFromSwiftError when backend supports swifterror");
4615
4616 assert(!I.isVolatile() &&
4617 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4618 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4619 "Support volatile, non temporal, invariant for load_from_swift_error");
4620
4621 const Value *SV = I.getOperand(0);
4622 Type *Ty = I.getType();
4623 assert(
4624 (!AA ||
4627 I.getAAMetadata()))) &&
4628 "load_from_swift_error should not be constant memory");
4629
4630 SmallVector<EVT, 4> ValueVTs;
4633 ValueVTs, &Offsets, 0);
4634 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4635 "expect a single EVT for swifterror");
4636
4637 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4639 getRoot(), getCurSDLoc(),
4640 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4641
4642 setValue(&I, L);
4643}
4644
4645void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4646 if (I.isAtomic())
4647 return visitAtomicStore(I);
4648
4649 const Value *SrcV = I.getOperand(0);
4650 const Value *PtrV = I.getOperand(1);
4651
4653 if (TLI.supportSwiftError()) {
4654 // Swifterror values can come from either a function parameter with
4655 // swifterror attribute or an alloca with swifterror attribute.
4656 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4657 if (Arg->hasSwiftErrorAttr())
4658 return visitStoreToSwiftError(I);
4659 }
4660
4661 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4662 if (Alloca->isSwiftError())
4663 return visitStoreToSwiftError(I);
4664 }
4665 }
4666
4667 SmallVector<EVT, 4> ValueVTs, MemVTs;
4670 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4671 unsigned NumValues = ValueVTs.size();
4672 if (NumValues == 0)
4673 return;
4674
4675 // Get the lowered operands. Note that we do this after
4676 // checking if NumResults is zero, because with zero results
4677 // the operands won't have values in the map.
4678 SDValue Src = getValue(SrcV);
4679 SDValue Ptr = getValue(PtrV);
4680
4681 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4682 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4683 SDLoc dl = getCurSDLoc();
4684 Align Alignment = I.getAlign();
4685 AAMDNodes AAInfo = I.getAAMetadata();
4686
4687 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4688
4689 unsigned ChainI = 0;
4690 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4691 // See visitLoad comments.
4692 if (ChainI == MaxParallelChains) {
4693 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4694 ArrayRef(Chains.data(), ChainI));
4695 Root = Chain;
4696 ChainI = 0;
4697 }
4698
4699 // TODO: MachinePointerInfo only supports a fixed length offset.
4700 MachinePointerInfo PtrInfo =
4701 !Offsets[i].isScalable() || Offsets[i].isZero()
4702 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4704
4705 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4706 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4707 if (MemVTs[i] != ValueVTs[i])
4708 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4709 SDValue St =
4710 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4711 Chains[ChainI] = St;
4712 }
4713
4714 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4715 ArrayRef(Chains.data(), ChainI));
4716 setValue(&I, StoreNode);
4717 DAG.setRoot(StoreNode);
4718}
4719
4720void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4721 bool IsCompressing) {
4722 SDLoc sdl = getCurSDLoc();
4723
4724 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4725 Align &Alignment) {
4726 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4727 Src0 = I.getArgOperand(0);
4728 Ptr = I.getArgOperand(1);
4729 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getAlignValue();
4730 Mask = I.getArgOperand(3);
4731 };
4732 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4733 Align &Alignment) {
4734 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4735 Src0 = I.getArgOperand(0);
4736 Ptr = I.getArgOperand(1);
4737 Mask = I.getArgOperand(2);
4738 Alignment = I.getParamAlign(1).valueOrOne();
4739 };
4740
4741 Value *PtrOperand, *MaskOperand, *Src0Operand;
4742 Align Alignment;
4743 if (IsCompressing)
4744 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4745 else
4746 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4747
4748 SDValue Ptr = getValue(PtrOperand);
4749 SDValue Src0 = getValue(Src0Operand);
4750 SDValue Mask = getValue(MaskOperand);
4751 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4752
4753 EVT VT = Src0.getValueType();
4754
4757 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4758 SDValue StoreNode =
4759 DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4760 ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4761 DAG.setRoot(StoreNode);
4762 setValue(&I, StoreNode);
4763}
4764
4765// Get a uniform base for the Gather/Scatter intrinsic.
4766// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4767// We try to represent it as a base pointer + vector of indices.
4768// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4769// The first operand of the GEP may be a single pointer or a vector of pointers
4770// Example:
4771// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4772// or
4773// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4774// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4775//
4776// When the first GEP operand is a single pointer - it is the uniform base we
4777// are looking for. If first operand of the GEP is a splat vector - we
4778// extract the splat value and use it as a uniform base.
4779// In all other cases the function returns 'false'.
4781 ISD::MemIndexType &IndexType, SDValue &Scale,
4782 SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4783 uint64_t ElemSize) {
4784 SelectionDAG& DAG = SDB->DAG;
4785 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4786 const DataLayout &DL = DAG.getDataLayout();
4787
4788 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4789
4790 // Handle splat constant pointer.
4791 if (auto *C = dyn_cast<Constant>(Ptr)) {
4792 C = C->getSplatValue();
4793 if (!C)
4794 return false;
4795
4796 Base = SDB->getValue(C);
4797
4798 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4799 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4800 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4801 IndexType = ISD::SIGNED_SCALED;
4802 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4803 return true;
4804 }
4805
4806 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4807 if (!GEP || GEP->getParent() != CurBB)
4808 return false;
4809
4810 if (GEP->getNumOperands() != 2)
4811 return false;
4812
4813 const Value *BasePtr = GEP->getPointerOperand();
4814 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4815
4816 // Make sure the base is scalar and the index is a vector.
4817 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4818 return false;
4819
4820 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4821 if (ScaleVal.isScalable())
4822 return false;
4823
4824 // Target may not support the required addressing mode.
4825 if (ScaleVal != 1 &&
4826 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
4827 return false;
4828
4829 Base = SDB->getValue(BasePtr);
4830 Index = SDB->getValue(IndexVal);
4831 IndexType = ISD::SIGNED_SCALED;
4832
4833 Scale =
4834 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4835 return true;
4836}
4837
4838void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4839 SDLoc sdl = getCurSDLoc();
4840
4841 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4842 const Value *Ptr = I.getArgOperand(1);
4843 SDValue Src0 = getValue(I.getArgOperand(0));
4844 SDValue Mask = getValue(I.getArgOperand(3));
4845 EVT VT = Src0.getValueType();
4846 Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4847 ->getMaybeAlignValue()
4848 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4850
4851 SDValue Base;
4852 SDValue Index;
4853 ISD::MemIndexType IndexType;
4854 SDValue Scale;
4855 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4856 I.getParent(), VT.getScalarStoreSize());
4857
4858 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4861 // TODO: Make MachineMemOperands aware of scalable
4862 // vectors.
4863 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
4864 if (!UniformBase) {
4866 Index = getValue(Ptr);
4867 IndexType = ISD::SIGNED_SCALED;
4868 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4869 }
4870
4871 EVT IdxVT = Index.getValueType();
4872 EVT EltTy = IdxVT.getVectorElementType();
4873 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4874 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4875 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4876 }
4877
4878 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4879 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4880 Ops, MMO, IndexType, false);
4881 DAG.setRoot(Scatter);
4882 setValue(&I, Scatter);
4883}
4884
4885void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4886 SDLoc sdl = getCurSDLoc();
4887
4888 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4889 Align &Alignment) {
4890 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4891 Ptr = I.getArgOperand(0);
4892 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getAlignValue();
4893 Mask = I.getArgOperand(2);
4894 Src0 = I.getArgOperand(3);
4895 };
4896 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4897 Align &Alignment) {
4898 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4899 Ptr = I.getArgOperand(0);
4900 Alignment = I.getParamAlign(0).valueOrOne();
4901 Mask = I.getArgOperand(1);
4902 Src0 = I.getArgOperand(2);
4903 };
4904
4905 Value *PtrOperand, *MaskOperand, *Src0Operand;
4906 Align Alignment;
4907 if (IsExpanding)
4908 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4909 else
4910 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4911
4912 SDValue Ptr = getValue(PtrOperand);
4913 SDValue Src0 = getValue(Src0Operand);
4914 SDValue Mask = getValue(MaskOperand);
4915 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4916
4917 EVT VT = Src0.getValueType();
4918 AAMDNodes AAInfo = I.getAAMetadata();
4919 const MDNode *Ranges = getRangeMetadata(I);
4920
4921 // Do not serialize masked loads of constant memory with anything.
4922 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4923 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4924
4925 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4926
4929 LocationSize::beforeOrAfterPointer(), Alignment, AAInfo, Ranges);
4930
4931 SDValue Load =
4932 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4933 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4934 if (AddToChain)
4935 PendingLoads.push_back(Load.getValue(1));
4936 setValue(&I, Load);
4937}
4938
4939void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4940 SDLoc sdl = getCurSDLoc();
4941
4942 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4943 const Value *Ptr = I.getArgOperand(0);
4944 SDValue Src0 = getValue(I.getArgOperand(3));
4945 SDValue Mask = getValue(I.getArgOperand(2));
4946
4948 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4949 Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4950 ->getMaybeAlignValue()
4951 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4952
4953 const MDNode *Ranges = getRangeMetadata(I);
4954
4955 SDValue Root = DAG.getRoot();
4956 SDValue Base;
4957 SDValue Index;
4958 ISD::MemIndexType IndexType;
4959 SDValue Scale;
4960 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4961 I.getParent(), VT.getScalarStoreSize());
4962 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4965 // TODO: Make MachineMemOperands aware of scalable
4966 // vectors.
4967 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
4968
4969 if (!UniformBase) {
4971 Index = getValue(Ptr);
4972 IndexType = ISD::SIGNED_SCALED;
4973 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4974 }
4975
4976 EVT IdxVT = Index.getValueType();
4977 EVT EltTy = IdxVT.getVectorElementType();
4978 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4979 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4980 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4981 }
4982
4983 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4984 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4985 Ops, MMO, IndexType, ISD::NON_EXTLOAD);
4986
4987 PendingLoads.push_back(Gather.getValue(1));
4988 setValue(&I, Gather);
4989}
4990
4991void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4992 SDLoc dl = getCurSDLoc();
4993 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4994 AtomicOrdering FailureOrdering = I.getFailureOrdering();
4995 SyncScope::ID SSID = I.getSyncScopeID();
4996
4997 SDValue InChain = getRoot();
4998
4999 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
5000 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5001
5004
5007 MachinePointerInfo(I.getPointerOperand()), Flags,
5009 AAMDNodes(), nullptr, SSID, SuccessOrdering, FailureOrdering);
5010
5012 dl, MemVT, VTs, InChain,
5013 getValue(I.getPointerOperand()),
5014 getValue(I.getCompareOperand()),
5015 getValue(I.getNewValOperand()), MMO);
5016
5017 SDValue OutChain = L.getValue(2);
5018
5019 setValue(&I, L);
5020 DAG.setRoot(OutChain);
5021}
5022
5023void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5024 SDLoc dl = getCurSDLoc();
5026 switch (I.getOperation()) {
5027 default: llvm_unreachable("Unknown atomicrmw operation");
5045 break;
5048 break;
5049 }
5050 AtomicOrdering Ordering = I.getOrdering();
5051 SyncScope::ID SSID = I.getSyncScopeID();
5052
5053 SDValue InChain = getRoot();
5054
5055 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5058
5061 MachinePointerInfo(I.getPointerOperand()), Flags,
5063 AAMDNodes(), nullptr, SSID, Ordering);
5064
5065 SDValue L =
5066 DAG.getAtomic(NT, dl, MemVT, InChain,
5067 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5068 MMO);
5069
5070 SDValue OutChain = L.getValue(1);
5071
5072 setValue(&I, L);
5073 DAG.setRoot(OutChain);
5074}
5075
5076void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5077 SDLoc dl = getCurSDLoc();
5079 SDValue Ops[3];
5080 Ops[0] = getRoot();
5081 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5083 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5085 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5086 setValue(&I, N);
5087 DAG.setRoot(N);
5088}
5089
5090void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5091 SDLoc dl = getCurSDLoc();
5092 AtomicOrdering Order = I.getOrdering();
5093 SyncScope::ID SSID = I.getSyncScopeID();
5094
5095 SDValue InChain = getRoot();
5096
5098 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5099 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5100
5101 if (!TLI.supportsUnalignedAtomics() &&
5102 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5103 report_fatal_error("Cannot generate unaligned atomic load");
5104
5106
5108 MachinePointerInfo(I.getPointerOperand()), Flags,
5109 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5110 nullptr, SSID, Order);
5111
5112 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5113
5114 SDValue Ptr = getValue(I.getPointerOperand());
5115 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
5116 Ptr, MMO);
5117
5118 SDValue OutChain = L.getValue(1);
5119 if (MemVT != VT)
5120 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5121
5122 setValue(&I, L);
5123 DAG.setRoot(OutChain);
5124}
5125
5126void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5127 SDLoc dl = getCurSDLoc();
5128
5129 AtomicOrdering Ordering = I.getOrdering();
5130 SyncScope::ID SSID = I.getSyncScopeID();
5131
5132 SDValue InChain = getRoot();
5133
5135 EVT MemVT =
5136 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5137
5138 if (!TLI.supportsUnalignedAtomics() &&
5139 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5140 report_fatal_error("Cannot generate unaligned atomic store");
5141
5143
5146 MachinePointerInfo(I.getPointerOperand()), Flags,
5147 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5148 nullptr, SSID, Ordering);
5149
5150 SDValue Val = getValue(I.getValueOperand());
5151 if (Val.getValueType() != MemVT)
5152 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5153 SDValue Ptr = getValue(I.getPointerOperand());
5154
5155 SDValue OutChain =
5156 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5157
5158 setValue(&I, OutChain);
5159 DAG.setRoot(OutChain);
5160}
5161
5162/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5163/// node.
5164void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5165 unsigned Intrinsic) {
5166 // Ignore the callsite's attributes. A specific call site may be marked with
5167 // readnone, but the lowering code will expect the chain based on the
5168 // definition.
5169 const Function *F = I.getCalledFunction();
5170 bool HasChain = !F->doesNotAccessMemory();
5171 bool OnlyLoad = HasChain && F->onlyReadsMemory();
5172
5173 // Build the operand list.
5175 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5176 if (OnlyLoad) {
5177 // We don't need to serialize loads against other loads.
5178 Ops.push_back(DAG.getRoot());
5179 } else {
5180 Ops.push_back(getRoot());
5181 }
5182 }
5183
5184 // Info is set by getTgtMemIntrinsic
5187 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
5189 Intrinsic);
5190
5191 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5192 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
5194 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
5196
5197 // Add all operands of the call to the operand list.
5198 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5199 const Value *Arg = I.getArgOperand(i);
5200 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5201 Ops.push_back(getValue(Arg));
5202 continue;
5203 }
5204
5205 // Use TargetConstant instead of a regular constant for immarg.
5206 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5207 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5208 assert(CI->getBitWidth() <= 64 &&
5209 "large intrinsic immediates not handled");
5210 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5211 } else {
5212 Ops.push_back(
5213 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5214 }
5215 }
5216
5217 SmallVector<EVT, 4> ValueVTs;
5218 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5219
5220 if (HasChain)
5221 ValueVTs.push_back(MVT::Other);
5222
5223 SDVTList VTs = DAG.getVTList(ValueVTs);
5224
5225 // Propagate fast-math-flags from IR to node(s).
5227 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5228 Flags.copyFMF(*FPMO);
5229 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5230
5231 // Create the node.
5233
5234 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5235 auto *Token = Bundle->Inputs[0].get();
5236 SDValue ConvControlToken = getValue(Token);
5237 assert(Ops.back().getValueType() != MVT::Glue &&
5238 "Did not expected another glue node here.");
5239 ConvControlToken =
5240 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5241 Ops.push_back(ConvControlToken);
5242 }
5243
5244 // In some cases, custom collection of operands from CallInst I may be needed.
5246 if (IsTgtIntrinsic) {
5247 // This is target intrinsic that touches memory
5248 //
5249 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
5250 // didn't yield anything useful.
5252 if (Info.ptrVal)
5253 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5254 else if (Info.fallbackAddressSpace)
5255 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5256 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
5257 Info.memVT, MPI, Info.align, Info.flags,
5258 Info.size, I.getAAMetadata());
5259 } else if (!HasChain) {
5261 } else if (!I.getType()->isVoidTy()) {
5263 } else {
5265 }
5266
5267 if (HasChain) {
5268 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
5269 if (OnlyLoad)
5270 PendingLoads.push_back(Chain);
5271 else
5272 DAG.setRoot(Chain);
5273 }
5274
5275 if (!I.getType()->isVoidTy()) {
5276 if (!isa<VectorType>(I.getType()))
5277 Result = lowerRangeToAssertZExt(DAG, I, Result);
5278
5279 MaybeAlign Alignment = I.getRetAlign();
5280
5281 // Insert `assertalign` node if there's an alignment.
5282 if (InsertAssertAlign && Alignment) {
5283 Result =
5284 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5285 }
5286
5287 setValue(&I, Result);
5288 }
5289}
5290
5291/// GetSignificand - Get the significand and build it into a floating-point
5292/// number with exponent of 1:
5293///
5294/// Op = (Op & 0x007fffff) | 0x3f800000;
5295///
5296/// where Op is the hexadecimal representation of floating point value.
5298 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5299 DAG.getConstant(0x007fffff, dl, MVT::i32));
5300 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5301 DAG.getConstant(0x3f800000, dl, MVT::i32));
5302 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5303}
5304
5305/// GetExponent - Get the exponent:
5306///
5307/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5308///
5309/// where Op is the hexadecimal representation of floating point value.
5311 const TargetLowering &TLI, const SDLoc &dl) {
5312 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5313 DAG.getConstant(0x7f800000, dl, MVT::i32));
5314 SDValue t1 = DAG.getNode(
5315 ISD::SRL, dl, MVT::i32, t0,
5316 DAG.getConstant(23, dl,
5317 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
5318 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5319 DAG.getConstant(127, dl, MVT::i32));
5320 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5321}
5322
5323/// getF32Constant - Get 32-bit floating point constant.
5325 const SDLoc &dl) {
5326 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5327 MVT::f32);
5328}
5329
5331 SelectionDAG &DAG) {
5332 // TODO: What fast-math-flags should be set on the floating-point nodes?
5333
5334 // IntegerPartOfX = ((int32_t)(t0);
5335 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5336
5337 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5338 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5339 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5340
5341 // IntegerPartOfX <<= 23;
5342 IntegerPartOfX =
5343 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5344 DAG.getConstant(23, dl,
5346 MVT::i32, DAG.getDataLayout())));
5347
5348 SDValue TwoToFractionalPartOfX;
5349 if (LimitFloatPrecision <= 6) {
5350 // For floating-point precision of 6:
5351 //
5352 // TwoToFractionalPartOfX =
5353 // 0.997535578f +
5354 // (0.735607626f + 0.252464424f * x) * x;
5355 //
5356 // error 0.0144103317, which is 6 bits
5357 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5358 getF32Constant(DAG, 0x3e814304, dl));
5359 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5360 getF32Constant(DAG, 0x3f3c50c8, dl));
5361 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5362 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5363 getF32Constant(DAG, 0x3f7f5e7e, dl));
5364 } else if (LimitFloatPrecision <= 12) {
5365 // For floating-point precision of 12:
5366 //
5367 // TwoToFractionalPartOfX =
5368 // 0.999892986f +
5369 // (0.696457318f +
5370 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5371 //
5372 // error 0.000107046256, which is 13 to 14 bits
5373 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5374 getF32Constant(DAG, 0x3da235e3, dl));
5375 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5376 getF32Constant(DAG, 0x3e65b8f3, dl));
5377 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5378 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5379 getF32Constant(DAG, 0x3f324b07, dl));
5380 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5381 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5382 getF32Constant(DAG, 0x3f7ff8fd, dl));
5383 } else { // LimitFloatPrecision <= 18
5384 // For floating-point precision of 18:
5385 //
5386 // TwoToFractionalPartOfX =
5387 // 0.999999982f +
5388 // (0.693148872f +
5389 // (0.240227044f +
5390 // (0.554906021e-1f +
5391 // (0.961591928e-2f +
5392 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5393 // error 2.47208000*10^(-7), which is better than 18 bits
5394 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5395 getF32Constant(DAG, 0x3924b03e, dl));
5396 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5397 getF32Constant(DAG, 0x3ab24b87, dl));
5398 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5399 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5400 getF32Constant(DAG, 0x3c1d8c17, dl));
5401 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5402 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5403 getF32Constant(DAG, 0x3d634a1d, dl));
5404 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5405 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5406 getF32Constant(DAG, 0x3e75fe14, dl));
5407 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5408 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5409 getF32Constant(DAG, 0x3f317234, dl));
5410 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5411 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5412 getF32Constant(DAG, 0x3f800000, dl));
5413 }
5414
5415 // Add the exponent into the result in integer domain.
5416 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5417 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5418 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5419}
5420
5421/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5422/// limited-precision mode.
5424 const TargetLowering &TLI, SDNodeFlags Flags) {
5425 if (Op.getValueType() == MVT::f32 &&
5427
5428 // Put the exponent in the right bit position for later addition to the
5429 // final result:
5430 //
5431 // t0 = Op * log2(e)
5432
5433 // TODO: What fast-math-flags should be set here?
5434 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5435 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5436 return getLimitedPrecisionExp2(t0, dl, DAG);
5437 }
5438
5439 // No special expansion.
5440 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5441}
5442
5443/// expandLog - Lower a log intrinsic. Handles the special sequences for
5444/// limited-precision mode.
5446 const TargetLowering &TLI, SDNodeFlags Flags) {
5447 // TODO: What fast-math-flags should be set on the floating-point nodes?
5448
5449 if (Op.getValueType() == MVT::f32 &&
5451 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5452
5453 // Scale the exponent by log(2).
5454 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5455 SDValue LogOfExponent =
5456 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5457 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5458
5459 // Get the significand and build it into a floating-point number with
5460 // exponent of 1.
5461 SDValue X = GetSignificand(DAG, Op1, dl);
5462
5463 SDValue LogOfMantissa;
5464 if (LimitFloatPrecision <= 6) {
5465 // For floating-point precision of 6:
5466 //
5467 // LogofMantissa =
5468 // -1.1609546f +
5469 // (1.4034025f - 0.23903021f * x) * x;
5470 //
5471 // error 0.0034276066, which is better than 8 bits
5472 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5473 getF32Constant(DAG, 0xbe74c456, dl));
5474 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5475 getF32Constant(DAG, 0x3fb3a2b1, dl));
5476 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5477 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5478 getF32Constant(DAG, 0x3f949a29, dl));
5479 } else if (LimitFloatPrecision <= 12) {
5480 // For floating-point precision of 12:
5481 //
5482 // LogOfMantissa =
5483 // -1.7417939f +
5484 // (2.8212026f +
5485 // (-1.4699568f +
5486 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5487 //
5488 // error 0.000061011436, which is 14 bits
5489 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5490 getF32Constant(DAG, 0xbd67b6d6, dl));
5491 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5492 getF32Constant(DAG, 0x3ee4f4b8, dl));
5493 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5494 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5495 getF32Constant(DAG, 0x3fbc278b, dl));
5496 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5497 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5498 getF32Constant(DAG, 0x40348e95, dl));
5499 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5500 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5501 getF32Constant(DAG, 0x3fdef31a, dl));
5502 } else { // LimitFloatPrecision <= 18
5503 // For floating-point precision of 18:
5504 //
5505 // LogOfMantissa =
5506 // -2.1072184f +
5507 // (4.2372794f +
5508 // (-3.7029485f +
5509 // (2.2781945f +
5510 // (-0.87823314f +
5511 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5512 //
5513 // error 0.0000023660568, which is better than 18 bits
5514 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5515 getF32Constant(DAG, 0xbc91e5ac, dl));
5516 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5517 getF32Constant(DAG, 0x3e4350aa, dl));
5518 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5519 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5520 getF32Constant(DAG, 0x3f60d3e3, dl));
5521 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5522 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5523 getF32Constant(DAG, 0x4011cdf0, dl));
5524 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5525 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5526 getF32Constant(DAG, 0x406cfd1c, dl));
5527 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5528 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5529 getF32Constant(DAG, 0x408797cb, dl));
5530 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5531 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5532 getF32Constant(DAG, 0x4006dcab, dl));
5533 }
5534
5535 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5536 }
5537
5538 // No special expansion.
5539 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5540}
5541
5542/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5543/// limited-precision mode.
5545 const TargetLowering &TLI, SDNodeFlags Flags) {
5546 // TODO: What fast-math-flags should be set on the floating-point nodes?
5547
5548 if (Op.getValueType() == MVT::f32 &&
5550 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5551
5552 // Get the exponent.
5553 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5554
5555 // Get the significand and build it into a floating-point number with
5556 // exponent of 1.
5557 SDValue X = GetSignificand(DAG, Op1, dl);
5558
5559 // Different possible minimax approximations of significand in
5560 // floating-point for various degrees of accuracy over [1,2].
5561 SDValue Log2ofMantissa;
5562 if (LimitFloatPrecision <= 6) {
5563 // For floating-point precision of 6:
5564 //
5565 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5566 //
5567 // error 0.0049451742, which is more than 7 bits
5568 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5569 getF32Constant(DAG, 0xbeb08fe0, dl));
5570 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5571 getF32Constant(DAG, 0x40019463, dl));
5572 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5573 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5574 getF32Constant(DAG, 0x3fd6633d, dl));
5575 } else if (LimitFloatPrecision <= 12) {
5576 // For floating-point precision of 12:
5577 //
5578 // Log2ofMantissa =
5579 // -2.51285454f +
5580 // (4.07009056f +
5581 // (-2.12067489f +
5582 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5583 //
5584 // error 0.0000876136000, which is better than 13 bits
5585 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5586 getF32Constant(DAG, 0xbda7262e, dl));
5587 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5588 getF32Constant(DAG, 0x3f25280b, dl));
5589 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5590 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5591 getF32Constant(DAG, 0x4007b923, dl));
5592 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5593 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5594 getF32Constant(DAG, 0x40823e2f, dl));
5595 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5596 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5597 getF32Constant(DAG, 0x4020d29c, dl));
5598 } else { // LimitFloatPrecision <= 18
5599 // For floating-point precision of 18:
5600 //
5601 // Log2ofMantissa =
5602 // -3.0400495f +
5603 // (6.1129976f +
5604 // (-5.3420409f +
5605 // (3.2865683f +
5606 // (-1.2669343f +
5607 // (0.27515199f -
5608 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5609 //
5610 // error 0.0000018516, which is better than 18 bits
5611 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5612 getF32Constant(DAG, 0xbcd2769e, dl));
5613 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5614 getF32Constant(DAG, 0x3e8ce0b9, dl));
5615 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5616 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5617 getF32Constant(DAG, 0x3fa22ae7, dl));
5618 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5619 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5620 getF32Constant(DAG, 0x40525723, dl));
5621 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5622 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5623 getF32Constant(DAG, 0x40aaf200, dl));
5624 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5625 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5626 getF32Constant(DAG, 0x40c39dad, dl));
5627 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5628 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5629 getF32Constant(DAG, 0x4042902c, dl));
5630 }
5631
5632 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5633 }
5634
5635 // No special expansion.
5636 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5637}
5638
5639/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5640/// limited-precision mode.
5642 const TargetLowering &TLI, SDNodeFlags Flags) {
5643 // TODO: What fast-math-flags should be set on the floating-point nodes?
5644
5645 if (Op.getValueType() == MVT::f32 &&
5647 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5648
5649 // Scale the exponent by log10(2) [0.30102999f].
5650 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5651 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5652 getF32Constant(DAG, 0x3e9a209a, dl));
5653
5654 // Get the significand and build it into a floating-point number with
5655 // exponent of 1.
5656 SDValue X = GetSignificand(DAG, Op1, dl);
5657
5658 SDValue Log10ofMantissa;
5659 if (LimitFloatPrecision <= 6) {
5660 // For floating-point precision of 6:
5661 //
5662 // Log10ofMantissa =
5663 // -0.50419619f +
5664 // (0.60948995f - 0.10380950f * x) * x;
5665 //
5666 // error 0.0014886165, which is 6 bits
5667 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5668 getF32Constant(DAG, 0xbdd49a13, dl));
5669 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5670 getF32Constant(DAG, 0x3f1c0789, dl));
5671 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5672 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5673 getF32Constant(DAG, 0x3f011300, dl));
5674 } else if (LimitFloatPrecision <= 12) {
5675 // For floating-point precision of 12:
5676 //
5677 // Log10ofMantissa =
5678 // -0.64831180f +
5679 // (0.91751397f +
5680 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5681 //
5682 // error 0.00019228036, which is better than 12 bits
5683 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5684 getF32Constant(DAG, 0x3d431f31, dl));
5685 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5686 getF32Constant(DAG, 0x3ea21fb2, dl));
5687 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5688 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5689 getF32Constant(DAG, 0x3f6ae232, dl));
5690 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5691 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5692 getF32Constant(DAG, 0x3f25f7c3, dl));
5693 } else { // LimitFloatPrecision <= 18
5694 // For floating-point precision of 18:
5695 //
5696 // Log10ofMantissa =
5697 // -0.84299375f +
5698 // (1.5327582f +
5699 // (-1.0688956f +
5700 // (0.49102474f +
5701 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5702 //
5703 // error 0.0000037995730, which is better than 18 bits
5704 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5705 getF32Constant(DAG, 0x3c5d51ce, dl));
5706 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5707 getF32Constant(DAG, 0x3e00685a, dl));
5708 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5709 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5710 getF32Constant(DAG, 0x3efb6798, dl));
5711 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5712 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5713 getF32Constant(DAG, 0x3f88d192, dl));
5714 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5715 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5716 getF32Constant(DAG, 0x3fc4316c, dl));
5717 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5718 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5719 getF32Constant(DAG, 0x3f57ce70, dl));
5720 }
5721
5722 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5723 }
5724
5725 // No special expansion.
5726 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5727}
5728
5729/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5730/// limited-precision mode.
5732 const TargetLowering &TLI, SDNodeFlags Flags) {
5733 if (Op.getValueType() == MVT::f32 &&
5735 return getLimitedPrecisionExp2(Op, dl, DAG);
5736
5737 // No special expansion.
5738 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5739}
5740
5741/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5742/// limited-precision mode with x == 10.0f.
5743static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5744 SelectionDAG &DAG, const TargetLowering &TLI,
5745 SDNodeFlags Flags) {
5746 bool IsExp10 = false;
5747 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5749 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5750 APFloat Ten(10.0f);
5751 IsExp10 = LHSC->isExactlyValue(Ten);
5752 }
5753 }
5754
5755 // TODO: What fast-math-flags should be set on the FMUL node?
5756 if (IsExp10) {
5757 // Put the exponent in the right bit position for later addition to the
5758 // final result:
5759 //
5760 // #define LOG2OF10 3.3219281f
5761 // t0 = Op * LOG2OF10;
5762 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5763 getF32Constant(DAG, 0x40549a78, dl));
5764 return getLimitedPrecisionExp2(t0, dl, DAG);
5765 }
5766
5767 // No special expansion.
5768 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5769}
5770
5771/// ExpandPowI - Expand a llvm.powi intrinsic.
5772static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5773 SelectionDAG &DAG) {
5774 // If RHS is a constant, we can expand this out to a multiplication tree if
5775 // it's beneficial on the target, otherwise we end up lowering to a call to
5776 // __powidf2 (for example).
5777 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5778 unsigned Val = RHSC->getSExtValue();
5779
5780 // powi(x, 0) -> 1.0
5781 if (Val == 0)
5782 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5783
5785 Val, DAG.shouldOptForSize())) {
5786 // Get the exponent as a positive value.
5787 if ((int)Val < 0)
5788 Val = -Val;
5789 // We use the simple binary decomposition method to generate the multiply
5790 // sequence. There are more optimal ways to do this (for example,
5791 // powi(x,15) generates one more multiply than it should), but this has
5792 // the benefit of being both really simple and much better than a libcall.
5793 SDValue Res; // Logically starts equal to 1.0
5794 SDValue CurSquare = LHS;
5795 // TODO: Intrinsics should have fast-math-flags that propagate to these
5796 // nodes.
5797 while (Val) {
5798 if (Val & 1) {
5799 if (Res.getNode())
5800 Res =
5801 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5802 else
5803 Res = CurSquare; // 1.0*CurSquare.
5804 }
5805
5806 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5807 CurSquare, CurSquare);
5808 Val >>= 1;
5809 }
5810
5811 // If the original was negative, invert the result, producing 1/(x*x*x).
5812 if (RHSC->getSExtValue() < 0)
5813 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5814 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5815 return Res;
5816 }
5817 }
5818
5819 // Otherwise, expand to a libcall.
5820 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5821}
5822
5823static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5824 SDValue LHS, SDValue RHS, SDValue Scale,
5825 SelectionDAG &DAG, const TargetLowering &TLI) {
5826 EVT VT = LHS.getValueType();
5827 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5828 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5829 LLVMContext &Ctx = *DAG.getContext();
5830
5831 // If the type is legal but the operation isn't, this node might survive all
5832 // the way to operation legalization. If we end up there and we do not have
5833 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5834 // node.
5835
5836 // Coax the legalizer into expanding the node during type legalization instead
5837 // by bumping the size by one bit. This will force it to Promote, enabling the
5838 // early expansion and avoiding the need to expand later.
5839
5840 // We don't have to do this if Scale is 0; that can always be expanded, unless
5841 // it's a saturating signed operation. Those can experience true integer
5842 // division overflow, a case which we must avoid.
5843
5844 // FIXME: We wouldn't have to do this (or any of the early
5845 // expansion/promotion) if it was possible to expand a libcall of an
5846 // illegal type during operation legalization. But it's not, so things
5847 // get a bit hacky.
5848 unsigned ScaleInt = Scale->getAsZExtVal();
5849 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5850 (TLI.isTypeLegal(VT) ||
5851 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5853 Opcode, VT, ScaleInt);
5854 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5855 EVT PromVT;
5856 if (VT.isScalarInteger())
5857 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5858 else if (VT.isVector()) {
5859 PromVT = VT.getVectorElementType();
5860 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5861 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5862 } else
5863 llvm_unreachable("Wrong VT for DIVFIX?");
5864 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
5865 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
5866 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5867 // For saturating operations, we need to shift up the LHS to get the
5868 // proper saturation width, and then shift down again afterwards.
5869 if (Saturating)
5870 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5871 DAG.getConstant(1, DL, ShiftTy));
5872 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5873 if (Saturating)
5874 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5875 DAG.getConstant(1, DL, ShiftTy));
5876 return DAG.getZExtOrTrunc(Res, DL, VT);
5877 }
5878 }
5879
5880 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5881}
5882
5883// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5884// bitcasted, or split argument. Returns a list of <Register, size in bits>
5885static void
5886getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5887 const SDValue &N) {
5888 switch (N.getOpcode()) {
5889 case ISD::CopyFromReg: {
5890 SDValue Op = N.getOperand(1);
5891 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5892 Op.getValueType().getSizeInBits());
5893 return;
5894 }
5895 case ISD::BITCAST:
5896 case ISD::AssertZext:
5897 case ISD::AssertSext:
5898 case ISD::TRUNCATE:
5899 getUnderlyingArgRegs(Regs, N.getOperand(0));
5900 return;
5901 case ISD::BUILD_PAIR:
5902 case ISD::BUILD_VECTOR:
5904 for (SDValue Op : N->op_values())
5905 getUnderlyingArgRegs(Regs, Op);
5906 return;
5907 default:
5908 return;
5909 }
5910}
5911
5912/// If the DbgValueInst is a dbg_value of a function argument, create the
5913/// corresponding DBG_VALUE machine instruction for it now. At the end of
5914/// instruction selection, they will be inserted to the entry BB.
5915/// We don't currently support this for variadic dbg_values, as they shouldn't
5916/// appear for function arguments or in the prologue.
5917bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5918 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5919 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5920 const Argument *Arg = dyn_cast<Argument>(V);
5921 if (!Arg)
5922 return false;
5923
5926
5927 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5928 // we've been asked to pursue.
5929 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5930 bool Indirect) {
5931 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5932 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5933 // pointing at the VReg, which will be patched up later.
5934 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5936 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
5937 /* isKill */ false, /* isDead */ false,
5938 /* isUndef */ false, /* isEarlyClobber */ false,
5939 /* SubReg */ 0, /* isDebug */ true)});
5940
5941 auto *NewDIExpr = FragExpr;
5942 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5943 // the DIExpression.
5944 if (Indirect)
5945 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5947 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
5948 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
5949 } else {
5950 // Create a completely standard DBG_VALUE.
5951 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5952 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5953 }
5954 };
5955
5956 if (Kind == FuncArgumentDbgValueKind::Value) {
5957 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5958 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5959 // the entry block.
5960 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5961 if (!IsInEntryBlock)
5962 return false;
5963
5964 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5965 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5966 // variable that also is a param.
5967 //
5968 // Although, if we are at the top of the entry block already, we can still
5969 // emit using ArgDbgValue. This might catch some situations when the
5970 // dbg.value refers to an argument that isn't used in the entry block, so
5971 // any CopyToReg node would be optimized out and the only way to express
5972 // this DBG_VALUE is by using the physical reg (or FI) as done in this
5973 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
5974 // we should only emit as ArgDbgValue if the Variable is an argument to the
5975 // current function, and the dbg.value intrinsic is found in the entry
5976 // block.
5977 bool VariableIsFunctionInputArg = Variable->isParameter() &&
5978 !DL->getInlinedAt();
5979 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5980 if (!IsInPrologue && !VariableIsFunctionInputArg)
5981 return false;
5982
5983 // Here we assume that a function argument on IR level only can be used to
5984 // describe one input parameter on source level. If we for example have
5985 // source code like this
5986 //
5987 // struct A { long x, y; };
5988 // void foo(struct A a, long b) {
5989 // ...
5990 // b = a.x;
5991 // ...
5992 // }
5993 //
5994 // and IR like this
5995 //
5996 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
5997 // entry:
5998 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
5999 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6000 // call void @llvm.dbg.value(metadata i32 %b, "b",
6001 // ...
6002 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6003 // ...
6004 //
6005 // then the last dbg.value is describing a parameter "b" using a value that
6006 // is an argument. But since we already has used %a1 to describe a parameter
6007 // we should not handle that last dbg.value here (that would result in an
6008 // incorrect hoisting of the DBG_VALUE to the function entry).
6009 // Notice that we allow one dbg.value per IR level argument, to accommodate
6010 // for the situation with fragments above.
6011 if (VariableIsFunctionInputArg) {
6012 unsigned ArgNo = Arg->getArgNo();
6013 if (ArgNo >= FuncInfo.DescribedArgs.size())
6014 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6015 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6016 return false;
6017 FuncInfo.DescribedArgs.set(ArgNo);
6018 }
6019 }
6020
6021 bool IsIndirect = false;
6022 std::optional<MachineOperand> Op;
6023 // Some arguments' frame index is recorded during argument lowering.
6024 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6025 if (FI != std::numeric_limits<int>::max())
6027
6029 if (!Op && N.getNode()) {
6030 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6031 Register Reg;
6032 if (ArgRegsAndSizes.size() == 1)
6033 Reg = ArgRegsAndSizes.front().first;
6034
6035 if (Reg && Reg.isVirtual()) {
6037 Register PR = RegInfo.getLiveInPhysReg(Reg);
6038 if (PR)
6039 Reg = PR;
6040 }
6041 if (Reg) {
6042 Op = MachineOperand::CreateReg(Reg, false);
6043 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6044 }
6045 }
6046
6047 if (!Op && N.getNode()) {
6048 // Check if frame index is available.
6049 SDValue LCandidate = peekThroughBitcasts(N);
6050 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6051 if (FrameIndexSDNode *FINode =
6052 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6053 Op = MachineOperand::CreateFI(FINode->getIndex());
6054 }
6055
6056 if (!Op) {
6057 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6058 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
6059 SplitRegs) {
6060 unsigned Offset = 0;
6061 for (const auto &RegAndSize : SplitRegs) {
6062 // If the expression is already a fragment, the current register
6063 // offset+size might extend beyond the fragment. In this case, only
6064 // the register bits that are inside the fragment are relevant.
6065 int RegFragmentSizeInBits = RegAndSize.second;
6066 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6067 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6068 // The register is entirely outside the expression fragment,
6069 // so is irrelevant for debug info.
6070 if (Offset >= ExprFragmentSizeInBits)
6071 break;
6072 // The register is partially outside the expression fragment, only
6073 // the low bits within the fragment are relevant for debug info.
6074 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6075 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6076 }
6077 }
6078
6079 auto FragmentExpr = DIExpression::createFragmentExpression(
6080 Expr, Offset, RegFragmentSizeInBits);
6081 Offset += RegAndSize.second;
6082 // If a valid fragment expression cannot be created, the variable's
6083 // correct value cannot be determined and so it is set as Undef.
6084 if (!FragmentExpr) {
6086 Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
6087 DAG.AddDbgValue(SDV, false);
6088 continue;
6089 }
6090 MachineInstr *NewMI =
6091 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6092 Kind != FuncArgumentDbgValueKind::Value);
6093 FuncInfo.ArgDbgValues.push_back(NewMI);
6094 }
6095 };
6096
6097 // Check if ValueMap has reg number.
6099 VMI = FuncInfo.ValueMap.find(V);
6100 if (VMI != FuncInfo.ValueMap.end()) {
6101 const auto &TLI = DAG.getTargetLoweringInfo();
6102 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6103 V->getType(), std::nullopt);
6104 if (RFV.occupiesMultipleRegs()) {
6105 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6106 return true;
6107 }
6108
6109 Op = MachineOperand::CreateReg(VMI->second, false);
6110 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6111 } else if (ArgRegsAndSizes.size() > 1) {
6112 // This was split due to the calling convention, and no virtual register
6113 // mapping exists for the value.
6114 splitMultiRegDbgValue(ArgRegsAndSizes);
6115 return true;
6116 }
6117 }
6118
6119 if (!Op)
6120 return false;
6121
6123 "Expected inlined-at fields to agree");
6124 MachineInstr *NewMI = nullptr;
6125
6126 if (Op->isReg())
6127 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6128 else
6129 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6130 Variable, Expr);
6131
6132 // Otherwise, use ArgDbgValues.
6133 FuncInfo.ArgDbgValues.push_back(NewMI);
6134 return true;
6135}
6136
6137/// Return the appropriate SDDbgValue based on N.
6138SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6139 DILocalVariable *Variable,
6140 DIExpression *Expr,
6141 const DebugLoc &dl,
6142 unsigned DbgSDNodeOrder) {
6143 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6144 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6145 // stack slot locations.
6146 //
6147 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6148 // debug values here after optimization:
6149 //
6150 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6151 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6152 //
6153 // Both describe the direct values of their associated variables.
6154 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6155 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6156 }
6157 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6158 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6159}
6160
6161static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6162 switch (Intrinsic) {
6163 case Intrinsic::smul_fix:
6164 return ISD::SMULFIX;
6165 case Intrinsic::umul_fix:
6166 return ISD::UMULFIX;
6167 case Intrinsic::smul_fix_sat:
6168 return ISD::SMULFIXSAT;
6169 case Intrinsic::umul_fix_sat:
6170 return ISD::UMULFIXSAT;
6171 case Intrinsic::sdiv_fix:
6172 return ISD::SDIVFIX;
6173 case Intrinsic::udiv_fix:
6174 return ISD::UDIVFIX;
6175 case Intrinsic::sdiv_fix_sat:
6176 return ISD::SDIVFIXSAT;
6177 case Intrinsic::udiv_fix_sat:
6178 return ISD::UDIVFIXSAT;
6179 default:
6180 llvm_unreachable("Unhandled fixed point intrinsic");
6181 }
6182}
6183
6184void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
6185 const char *FunctionName) {
6186 assert(FunctionName && "FunctionName must not be nullptr");
6188 FunctionName,
6190 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
6191}
6192
6193/// Given a @llvm.call.preallocated.setup, return the corresponding
6194/// preallocated call.
6195static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6196 assert(cast<CallBase>(PreallocatedSetup)
6198 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6199 "expected call_preallocated_setup Value");
6200 for (const auto *U : PreallocatedSetup->users()) {
6201 auto *UseCall = cast<CallBase>(U);
6202 const Function *Fn = UseCall->getCalledFunction();
6203 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6204 return UseCall;
6205 }
6206 }
6207 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6208}
6209
6210/// If DI is a debug value with an EntryValue expression, lower it using the
6211/// corresponding physical register of the associated Argument value
6212/// (guaranteed to exist by the verifier).
6213bool SelectionDAGBuilder::visitEntryValueDbgValue(
6214 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6215 DIExpression *Expr, DebugLoc DbgLoc) {
6216 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6217 return false;
6218
6219 // These properties are guaranteed by the verifier.
6220 const Argument *Arg = cast<Argument>(Values[0]);
6221 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6222
6223 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6224 if (ArgIt == FuncInfo.ValueMap.end()) {
6225 LLVM_DEBUG(
6226 dbgs() << "Dropping dbg.value: expression is entry_value but "
6227 "couldn't find an associated register for the Argument\n");
6228 return true;
6229 }
6230 Register ArgVReg = ArgIt->getSecond();
6231
6232 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6233 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6235 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6236 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6237 return true;
6238 }
6239 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6240 "couldn't find a physical register\n");
6241 return true;
6242}
6243
6244/// Lower the call to the specified intrinsic function.
6245void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6246 unsigned Intrinsic) {
6247 SDLoc sdl = getCurSDLoc();
6248 switch (Intrinsic) {
6249 case Intrinsic::experimental_convergence_anchor:
6250 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6251 break;
6252 case Intrinsic::experimental_convergence_entry:
6253 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6254 break;
6255 case Intrinsic::experimental_convergence_loop: {
6256 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6257 auto *Token = Bundle->Inputs[0].get();
6258 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6259 getValue(Token)));
6260 break;
6261 }
6262 }
6263}
6264
6265/// Lower the call to the specified intrinsic function.
6266void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6267 unsigned Intrinsic) {
6269 SDLoc sdl = getCurSDLoc();
6270 DebugLoc dl = getCurDebugLoc();
6271 SDValue Res;
6272
6274 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6275 Flags.copyFMF(*FPOp);
6276
6277 switch (Intrinsic) {
6278 default:
6279 // By default, turn this into a target intrinsic node.
6280 visitTargetIntrinsic(I, Intrinsic);
6281 return;
6282 case Intrinsic::vscale: {
6283 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6284 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6285 return;
6286 }
6287 case Intrinsic::vastart: visitVAStart(I); return;
6288 case Intrinsic::vaend: visitVAEnd(I); return;
6289 case Intrinsic::vacopy: visitVACopy(I); return;
6290 case Intrinsic::returnaddress:
6292 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6293 getValue(I.getArgOperand(0))));
6294 return;
6295 case Intrinsic::addressofreturnaddress:
6296 setValue(&I,
6298 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6299 return;
6300 case Intrinsic::sponentry:
6301 setValue(&I,
6303 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6304 return;
6305 case Intrinsic::frameaddress:
6308 getValue(I.getArgOperand(0))));
6309 return;
6310 case Intrinsic::read_volatile_register:
6311 case Intrinsic::read_register: {
6312 Value *Reg = I.getArgOperand(0);
6313 SDValue Chain = getRoot();
6315 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6316 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6317 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6318 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6319 setValue(&I, Res);
6320 DAG.setRoot(Res.getValue(1));
6321 return;
6322 }
6323 case Intrinsic::write_register: {
6324 Value *Reg = I.getArgOperand(0);
6325 Value *RegValue = I.getArgOperand(1);
6326 SDValue Chain = getRoot();
6328 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6329 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6330 RegName, getValue(RegValue)));
6331 return;
6332 }
6333 case Intrinsic::memcpy: {
6334 const auto &MCI = cast<MemCpyInst>(I);
6335 SDValue Op1 = getValue(I.getArgOperand(0));
6336 SDValue Op2 = getValue(I.getArgOperand(1));
6337 SDValue Op3 = getValue(I.getArgOperand(2));
6338 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
6339 Align DstAlign = MCI.getDestAlign().valueOrOne();
6340 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6341 Align Alignment = std::min(DstAlign, SrcAlign);
6342 bool isVol = MCI.isVolatile();
6343 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6344 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6345 // node.
6346 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6347 SDValue MC = DAG.getMemcpy(
6348 Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6349 /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
6350 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
6351 updateDAGForMaybeTailCall(MC);
6352 return;
6353 }
6354 case Intrinsic::memcpy_inline: {
6355 const auto &MCI = cast<MemCpyInlineInst>(I);
6356 SDValue Dst = getValue(I.getArgOperand(0));
6357 SDValue Src = getValue(I.getArgOperand(1));
6358 SDValue Size = getValue(I.getArgOperand(2));
6359 assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
6360 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6361 Align DstAlign = MCI.getDestAlign().valueOrOne();
6362 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6363 Align Alignment = std::min(DstAlign, SrcAlign);
6364 bool isVol = MCI.isVolatile();
6365 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6366 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6367 // node.
6368 SDValue MC = DAG.getMemcpy(
6369 getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
6370 /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
6371 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
6372 updateDAGForMaybeTailCall(MC);
6373 return;
6374 }
6375 case Intrinsic::memset: {
6376 const auto &MSI = cast<MemSetInst>(I);
6377 SDValue Op1 = getValue(I.getArgOperand(0));
6378 SDValue Op2 = getValue(I.getArgOperand(1));
6379 SDValue Op3 = getValue(I.getArgOperand(2));
6380 // @llvm.memset defines 0 and 1 to both mean no alignment.
6381 Align Alignment = MSI.getDestAlign().valueOrOne();
6382 bool isVol = MSI.isVolatile();
6383 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6384 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6385 SDValue MS = DAG.getMemset(
6386 Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
6387 isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6388 updateDAGForMaybeTailCall(MS);
6389 return;
6390 }
6391 case Intrinsic::memset_inline: {
6392 const auto &MSII = cast<MemSetInlineInst>(I);
6393 SDValue Dst = getValue(I.getArgOperand(0));
6394 SDValue Value = getValue(I.getArgOperand(1));
6395 SDValue Size = getValue(I.getArgOperand(2));
6396 assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
6397 // @llvm.memset defines 0 and 1 to both mean no alignment.
6398 Align DstAlign = MSII.getDestAlign().valueOrOne();
6399 bool isVol = MSII.isVolatile();
6400 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6401 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6402 SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
6403 /* AlwaysInline */ true, isTC,
6404 MachinePointerInfo(I.getArgOperand(0)),
6405 I.getAAMetadata());
6406 updateDAGForMaybeTailCall(MC);
6407 return;
6408 }
6409 case Intrinsic::memmove: {
6410 const auto &MMI = cast<MemMoveInst>(I);
6411 SDValue Op1 = getValue(I.getArgOperand(0));
6412 SDValue Op2 = getValue(I.getArgOperand(1));
6413 SDValue Op3 = getValue(I.getArgOperand(2));
6414 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6415 Align DstAlign = MMI.getDestAlign().valueOrOne();
6416 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6417 Align Alignment = std::min(DstAlign, SrcAlign);
6418 bool isVol = MMI.isVolatile();
6419 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6420 // FIXME: Support passing different dest/src alignments to the memmove DAG
6421 // node.
6422 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6423 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6424 isTC, MachinePointerInfo(I.getArgOperand(0)),
6425 MachinePointerInfo(I.getArgOperand(1)),
6426 I.getAAMetadata(), AA);
6427 updateDAGForMaybeTailCall(MM);
6428 return;
6429 }
6430 case Intrinsic::memcpy_element_unordered_atomic: {
6431 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
6432 SDValue Dst = getValue(MI.getRawDest());
6433 SDValue Src = getValue(MI.getRawSource());
6434 SDValue Length = getValue(MI.getLength());
6435
6436 Type *LengthTy = MI.getLength()->getType();
6437 unsigned ElemSz = MI.getElementSizeInBytes();
6438 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6439 SDValue MC =
6440 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6441 isTC, MachinePointerInfo(MI.getRawDest()),
6442 MachinePointerInfo(MI.getRawSource()));
6443 updateDAGForMaybeTailCall(MC);
6444 return;
6445 }
6446 case Intrinsic::memmove_element_unordered_atomic: {
6447 auto &MI = cast<AtomicMemMoveInst>(I);
6448 SDValue Dst = getValue(MI.getRawDest());
6449 SDValue Src = getValue(MI.getRawSource());
6450 SDValue Length = getValue(MI.getLength());
6451
6452 Type *LengthTy = MI.getLength()->getType();
6453 unsigned ElemSz = MI.getElementSizeInBytes();
6454 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6455 SDValue MC =
6456 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6457 isTC, MachinePointerInfo(MI.getRawDest()),
6458 MachinePointerInfo(MI.getRawSource()));
6459 updateDAGForMaybeTailCall(MC);
6460 return;
6461 }
6462 case Intrinsic::memset_element_unordered_atomic: {
6463 auto &MI = cast<AtomicMemSetInst>(I);
6464 SDValue Dst = getValue(MI.getRawDest());
6465 SDValue Val = getValue(MI.getValue());
6466 SDValue Length = getValue(MI.getLength());
6467
6468 Type *LengthTy = MI.getLength()->getType();
6469 unsigned ElemSz = MI.getElementSizeInBytes();
6470 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6471 SDValue MC =
6472 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
6473 isTC, MachinePointerInfo(MI.getRawDest()));
6474 updateDAGForMaybeTailCall(MC);
6475 return;
6476 }
6477 case Intrinsic::call_preallocated_setup: {
6478 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6479 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6480 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6481 getRoot(), SrcValue);
6482 setValue(&I, Res);
6483 DAG.setRoot(Res);
6484 return;
6485 }
6486 case Intrinsic::call_preallocated_arg: {
6487 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6488 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6489 SDValue Ops[3];
6490 Ops[0] = getRoot();
6491 Ops[1] = SrcValue;
6492 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6493 MVT::i32); // arg index
6494 SDValue Res = DAG.getNode(
6496 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6497 setValue(&I, Res);
6498 DAG.setRoot(Res.getValue(1));
6499 return;
6500 }
6501 case Intrinsic::dbg_declare: {
6502 const auto &DI = cast<DbgDeclareInst>(I);
6503 // Debug intrinsics are handled separately in assignment tracking mode.
6504 // Some intrinsics are handled right after Argument lowering.
6505 if (AssignmentTrackingEnabled ||
6507 return;
6508 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DI << "\n");
6509 DILocalVariable *Variable = DI.getVariable();
6510 DIExpression *Expression = DI.getExpression();
6512 // Assume dbg.declare can not currently use DIArgList, i.e.
6513 // it is non-variadic.
6514 assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6515 handleDebugDeclare(DI.getVariableLocationOp(0), Variable, Expression,
6516 DI.getDebugLoc());
6517 return;
6518 }
6519 case Intrinsic::dbg_label: {
6520 const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6521 DILabel *Label = DI.getLabel();
6522 assert(Label && "Missing label");
6523
6524 SDDbgLabel *SDV;
6525 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6526 DAG.AddDbgLabel(SDV);
6527 return;
6528 }
6529 case Intrinsic::dbg_assign: {
6530 // Debug intrinsics are handled seperately in assignment tracking mode.
6531 if (AssignmentTrackingEnabled)
6532 return;
6533 // If assignment tracking hasn't been enabled then fall through and treat
6534 // the dbg.assign as a dbg.value.
6535 [[fallthrough]];
6536 }
6537 case Intrinsic::dbg_value: {
6538 // Debug intrinsics are handled seperately in assignment tracking mode.
6539 if (AssignmentTrackingEnabled)
6540 return;
6541 const DbgValueInst &DI = cast<DbgValueInst>(I);
6542 assert(DI.getVariable() && "Missing variable");
6543
6544 DILocalVariable *Variable = DI.getVariable();
6547
6548 if (DI.isKillLocation()) {
6549 handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
6550 return;
6551 }
6552
6554 if (Values.empty())
6555 return;
6556
6557 bool IsVariadic = DI.hasArgList();
6558 if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
6559 SDNodeOrder, IsVariadic))
6560 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
6561 DI.getDebugLoc(), SDNodeOrder);
6562 return;
6563 }
6564
6565 case Intrinsic::eh_typeid_for: {
6566 // Find the type id for the given typeinfo.
6567 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6568 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6569 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6570 setValue(&I, Res);
6571 return;
6572 }
6573
6574 case Intrinsic::eh_return_i32:
6575 case Intrinsic::eh_return_i64:
6578 MVT::Other,
6580 getValue(I.getArgOperand(0)),
6581 getValue(I.getArgOperand(1))));
6582 return;
6583 case Intrinsic::eh_unwind_init:
6585 return;
6586 case Intrinsic::eh_dwarf_cfa:
6589 getValue(I.getArgOperand(0))));
6590 return;
6591 case Intrinsic::eh_sjlj_callsite: {
6593 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6594 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
6595
6597 return;
6598 }
6599 case Intrinsic::eh_sjlj_functioncontext: {
6600 // Get and store the index of the function context.
6602 AllocaInst *FnCtx =
6603 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6604 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6606 return;
6607 }
6608 case Intrinsic::eh_sjlj_setjmp: {
6609 SDValue Ops[2];
6610 Ops[0] = getRoot();
6611 Ops[1] = getValue(I.getArgOperand(0));
6613 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6614 setValue(&I, Op.getValue(0));
6615 DAG.setRoot(Op.getValue(1));
6616 return;
6617 }
6618 case Intrinsic::eh_sjlj_longjmp:
6619 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6620 getRoot(), getValue(I.getArgOperand(0))));
6621 return;
6622 case Intrinsic::eh_sjlj_setup_dispatch:
6624 getRoot()));
6625 return;
6626 case Intrinsic::masked_gather:
6627 visitMaskedGather(I);
6628 return;
6629 case Intrinsic::masked_load:
6630 visitMaskedLoad(I);
6631 return;
6632 case Intrinsic::masked_scatter:
6633 visitMaskedScatter(I);
6634 return;
6635 case Intrinsic::masked_store:
6636 visitMaskedStore(I);
6637 return;
6638 case Intrinsic::masked_expandload:
6639 visitMaskedLoad(I, true /* IsExpanding */);
6640 return;
6641 case Intrinsic::masked_compressstore:
6642 visitMaskedStore(I, true /* IsCompressing */);
6643 return;
6644 case Intrinsic::powi:
6645 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6646 getValue(I.getArgOperand(1)), DAG));
6647 return;
6648 case Intrinsic::log:
6649 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6650 return;
6651 case Intrinsic::log2:
6652 setValue(&I,
6653 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6654 return;
6655 case Intrinsic::log10:
6656 setValue(&I,
6657 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6658 return;
6659 case Intrinsic::exp:
6660 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6661 return;
6662 case Intrinsic::exp2:
6663 setValue(&I,
6664 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6665 return;
6666 case Intrinsic::pow:
6667 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6668 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6669 return;
6670 case Intrinsic::sqrt:
6671 case Intrinsic::fabs:
6672 case Intrinsic::sin:
6673 case Intrinsic::cos:
6674 case Intrinsic::exp10:
6675 case Intrinsic::floor:
6676 case Intrinsic::ceil:
6677 case Intrinsic::trunc:
6678 case Intrinsic::rint:
6679 case Intrinsic::nearbyint:
6680 case Intrinsic::round:
6681 case Intrinsic::roundeven:
6682 case Intrinsic::canonicalize: {
6683 unsigned Opcode;
6684 switch (Intrinsic) {
6685 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6686 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6687 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6688 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6689 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6690 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6691 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6692 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6693 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6694 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6695 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6696 case Intrinsic::round: Opcode = ISD::FROUND; break;
6697 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6698 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6699 }
6700
6701 setValue(&I, DAG.getNode(Opcode, sdl,
6702 getValue(I.getArgOperand(0)).getValueType(),
6703 getValue(I.getArgOperand(0)), Flags));
6704 return;
6705 }
6706 case Intrinsic::lround:
6707 case Intrinsic::llround:
6708 case Intrinsic::lrint:
6709 case Intrinsic::llrint: {
6710 unsigned Opcode;
6711 switch (Intrinsic) {
6712 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6713 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6714 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6715 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6716 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6717 }
6718
6719 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6720 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6721 getValue(I.getArgOperand(0))));
6722 return;
6723 }
6724 case Intrinsic::minnum:
6726 getValue(I.getArgOperand(0)).getValueType(),
6727 getValue(I.getArgOperand(0)),
6728 getValue(I.getArgOperand(1)), Flags));
6729 return;
6730 case Intrinsic::maxnum:
6732 getValue(I.getArgOperand(0)).getValueType(),
6733 getValue(I.getArgOperand(0)),
6734 getValue(I.getArgOperand(1)), Flags));
6735 return;
6736 case Intrinsic::minimum:
6738 getValue(I.getArgOperand(0)).getValueType(),
6739 getValue(I.getArgOperand(0)),
6740 getValue(I.getArgOperand(1)), Flags));
6741 return;
6742 case Intrinsic::maximum:
6744 getValue(I.getArgOperand(0)).getValueType(),
6745 getValue(I.getArgOperand(0)),
6746 getValue(I.getArgOperand(1)), Flags));
6747 return;
6748 case Intrinsic::copysign:
6750 getValue(I.getArgOperand(0)).getValueType(),
6751 getValue(I.getArgOperand(0)),
6752 getValue(I.getArgOperand(1)), Flags));
6753 return;
6754 case Intrinsic::ldexp:
6756 getValue(I.getArgOperand(0)).getValueType(),
6757 getValue(I.getArgOperand(0)),
6758 getValue(I.getArgOperand(1)), Flags));
6759 return;
6760 case Intrinsic::frexp: {
6761 SmallVector<EVT, 2> ValueVTs;
6762 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
6763 SDVTList VTs = DAG.getVTList(ValueVTs);
6764 setValue(&I,
6765 DAG.getNode(ISD::FFREXP, sdl, VTs, getValue(I.getArgOperand(0))));
6766 return;
6767 }
6768 case Intrinsic::arithmetic_fence: {
6770 getValue(I.getArgOperand(0)).getValueType(),
6771 getValue(I.getArgOperand(0)), Flags));
6772 return;
6773 }
6774 case Intrinsic::fma:
6776 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6777 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6778 getValue(I.getArgOperand(2)), Flags));
6779 return;
6780#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
6781 case Intrinsic::INTRINSIC:
6782#include "llvm/IR/ConstrainedOps.def"
6783 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6784 return;
6785#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6786#include "llvm/IR/VPIntrinsics.def"
6787 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6788 return;
6789 case Intrinsic::fptrunc_round: {
6790 // Get the last argument, the metadata and convert it to an integer in the
6791 // call
6792 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6793 std::optional<RoundingMode> RoundMode =
6794 convertStrToRoundingMode(cast<MDString>(MD)->getString());
6795
6796 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6797
6798 // Propagate fast-math-flags from IR to node(s).
6800 Flags.copyFMF(*cast<FPMathOperator>(&I));
6801 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6802
6804 Result = DAG.getNode(
6805 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6806 DAG.getTargetConstant((int)*RoundMode, sdl,
6808 setValue(&I, Result);
6809
6810 return;
6811 }
6812 case Intrinsic::fmuladd: {
6813 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6816 setValue(&I, DAG.getNode(ISD::FMA, sdl,
6817 getValue(I.getArgOperand(0)).getValueType(),
6818 getValue(I.getArgOperand(0)),
6819 getValue(I.getArgOperand(1)),
6820 getValue(I.getArgOperand(2)), Flags));
6821 } else {
6822 // TODO: Intrinsic calls should have fast-math-flags.
6824 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6825 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6827 getValue(I.getArgOperand(0)).getValueType(),
6828 Mul, getValue(I.getArgOperand(2)), Flags);
6829 setValue(&I, Add);
6830 }
6831 return;
6832 }
6833 case Intrinsic::convert_to_fp16:
6834 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6835 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6836 getValue(I.getArgOperand(0)),
6837 DAG.getTargetConstant(0, sdl,
6838 MVT::i32))));
6839 return;
6840 case Intrinsic::convert_from_fp16:
6842 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6843 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6844 getValue(I.getArgOperand(0)))));
6845 return;
6846 case Intrinsic::fptosi_sat: {
6847 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6849 getValue(I.getArgOperand(0)),
6851 return;
6852 }
6853 case Intrinsic::fptoui_sat: {
6854 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6856 getValue(I.getArgOperand(0)),
6858 return;
6859 }
6860 case Intrinsic::set_rounding:
6861 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
6862 {getRoot(), getValue(I.getArgOperand(0))});
6863 setValue(&I, Res);
6864 DAG.setRoot(Res.getValue(0));
6865 return;
6866 case Intrinsic::is_fpclass: {
6867 const DataLayout DLayout = DAG.getDataLayout();
6868 EVT DestVT = TLI.getValueType(DLayout, I.getType());
6869 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
6870 FPClassTest Test = static_cast<FPClassTest>(
6871 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
6873 const Function &F = MF.getFunction();
6874 SDValue Op = getValue(I.getArgOperand(0));
6876 Flags.setNoFPExcept(
6877 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
6878 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
6879 // expansion can use illegal types. Making expansion early allows
6880 // legalizing these types prior to selection.
6881 if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
6882 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
6883 setValue(&I, Result);
6884 return;
6885 }
6886
6887 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
6888 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
6889 setValue(&I, V);
6890 return;
6891 }
6892 case Intrinsic::get_fpenv: {
6893 const DataLayout DLayout = DAG.getDataLayout();
6894 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
6895 Align TempAlign = DAG.getEVTAlign(EnvVT);
6896 SDValue Chain = getRoot();
6897 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
6898 // and temporary storage in stack.
6899 if (TLI.isOperationLegalOrCustom(ISD::GET_FPENV, EnvVT)) {
6900 Res = DAG.getNode(
6901 ISD::GET_FPENV, sdl,
6902 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
6903 MVT::Other),
6904 Chain);
6905 } else {
6906 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
6907 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
6908 auto MPI =
6912 TempAlign);
6913 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
6914 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
6915 }
6916 setValue(&I, Res);
6917 DAG.setRoot(Res.getValue(1));
6918 return;
6919 }
6920 case Intrinsic::set_fpenv: {
6921 const DataLayout DLayout = DAG.getDataLayout();
6922 SDValue Env = getValue(I.getArgOperand(0));
6923 EVT EnvVT = Env.getValueType();
6924 Align TempAlign = DAG.getEVTAlign(EnvVT);
6925 SDValue Chain = getRoot();
6926 // If SET_FPENV is custom or legal, use it. Otherwise use loading
6927 // environment from memory.
6928 if (TLI.isOperationLegalOrCustom(ISD::SET_FPENV, EnvVT)) {
6929 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
6930 } else {
6931 // Allocate space in stack, copy environment bits into it and use this
6932 // memory in SET_FPENV_MEM.
6933 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
6934 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
6935 auto MPI =
6937 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
6941 TempAlign);
6942 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
6943 }
6944 DAG.setRoot(Chain);
6945 return;
6946 }
6947 case Intrinsic::reset_fpenv:
6948 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
6949 return;
6950 case Intrinsic::get_fpmode:
6951 Res = DAG.getNode(
6952 ISD::GET_FPMODE, sdl,
6953 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
6954 MVT::Other),
6955 DAG.getRoot());
6956 setValue(&I, Res);
6957 DAG.setRoot(Res.getValue(1));
6958 return;
6959 case Intrinsic::set_fpmode:
6960 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
6961 getValue(I.getArgOperand(0)));
6962 DAG.setRoot(Res);
6963 return;
6964 case Intrinsic::reset_fpmode: {
6965 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
6966 DAG.setRoot(Res);
6967 return;
6968 }
6969 case Intrinsic::pcmarker: {
6970 SDValue Tmp = getValue(I.getArgOperand(0));
6971 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
6972 return;
6973 }
6974 case Intrinsic::readcyclecounter: {
6975 SDValue Op = getRoot();
6977 DAG.getVTList(MVT::i64, MVT::Other), Op);
6978 setValue(&I, Res);
6979 DAG.setRoot(Res.getValue(1));
6980 return;
6981 }
6982 case Intrinsic::readsteadycounter: {
6983 SDValue Op = getRoot();
6985 DAG.getVTList(MVT::i64, MVT::Other), Op);
6986 setValue(&I, Res);
6987 DAG.setRoot(Res.getValue(1));
6988 return;
6989 }
6990 case Intrinsic::bitreverse:
6992 getValue(I.getArgOperand(0)).getValueType(),
6993 getValue(I.getArgOperand(0))));
6994 return;
6995 case Intrinsic::bswap:
6997 getValue(I.getArgOperand(0)).getValueType(),
6998 getValue(I.getArgOperand(0))));
6999 return;
7000 case Intrinsic::cttz: {
7001 SDValue Arg = getValue(I.getArgOperand(0));
7002 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7003 EVT Ty = Arg.getValueType();
7005 sdl, Ty, Arg));
7006 return;
7007 }
7008 case Intrinsic::ctlz: {
7009 SDValue Arg = getValue(I.getArgOperand(0));
7010 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7011 EVT Ty = Arg.getValueType();
7013 sdl, Ty, Arg));
7014 return;
7015 }
7016 case Intrinsic::ctpop: {
7017 SDValue Arg = getValue(I.getArgOperand(0));
7018 EVT Ty = Arg.getValueType();
7019 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7020 return;
7021 }
7022 case Intrinsic::fshl:
7023 case Intrinsic::fshr: {
7024 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7025 SDValue X = getValue(I.getArgOperand(0));
7026 SDValue Y = getValue(I.getArgOperand(1));
7027 SDValue Z = getValue(I.getArgOperand(2));
7028 EVT VT = X.getValueType();
7029
7030 if (X == Y) {
7031 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7032 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7033 } else {
7034 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7035 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7036 }
7037 return;
7038 }
7039 case Intrinsic::sadd_sat: {
7040 SDValue Op1 = getValue(I.getArgOperand(0));
7041 SDValue Op2 = getValue(I.getArgOperand(1));
7042 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7043 return;
7044 }
7045 case Intrinsic::uadd_sat: {
7046 SDValue Op1 = getValue(I.getArgOperand(0));
7047 SDValue Op2 = getValue(I.getArgOperand(1));
7048 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7049 return;
7050 }
7051 case Intrinsic::ssub_sat: {
7052 SDValue Op1 = getValue(I.getArgOperand(0));
7053 SDValue Op2 = getValue(I.getArgOperand(1));
7054 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7055 return;
7056 }
7057 case Intrinsic::usub_sat: {
7058 SDValue Op1 = getValue(I.getArgOperand(0));
7059 SDValue Op2 = getValue(I.getArgOperand(1));
7060 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7061 return;
7062 }
7063 case Intrinsic::sshl_sat: {
7064 SDValue Op1 = getValue(I.getArgOperand(0));
7065 SDValue Op2 = getValue(I.getArgOperand(1));
7066 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7067 return;
7068 }
7069 case Intrinsic::ushl_sat: {
7070 SDValue Op1 = getValue(I.getArgOperand(0));
7071 SDValue Op2 = getValue(I.getArgOperand(1));
7072 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7073 return;
7074 }
7075 case Intrinsic::smul_fix:
7076 case Intrinsic::umul_fix:
7077 case Intrinsic::smul_fix_sat:
7078 case Intrinsic::umul_fix_sat: {
7079 SDValue Op1 = getValue(I.getArgOperand(0));
7080 SDValue Op2 = getValue(I.getArgOperand(1));
7081 SDValue Op3 = getValue(I.getArgOperand(2));
7083 Op1.getValueType(), Op1, Op2, Op3));
7084 return;
7085 }
7086 case Intrinsic::sdiv_fix:
7087 case Intrinsic::udiv_fix:
7088 case Intrinsic::sdiv_fix_sat:
7089 case Intrinsic::udiv_fix_sat: {
7090 SDValue Op1 = getValue(I.getArgOperand(0));
7091 SDValue Op2 = getValue(I.getArgOperand(1));
7092 SDValue Op3 = getValue(I.getArgOperand(2));
7094 Op1, Op2, Op3, DAG, TLI));
7095 return;
7096 }
7097 case Intrinsic::smax: {
7098 SDValue Op1 = getValue(I.getArgOperand(0));
7099 SDValue Op2 = getValue(I.getArgOperand(1));
7100 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7101 return;
7102 }
7103 case Intrinsic::smin: {
7104 SDValue Op1 = getValue(I.getArgOperand(0));
7105 SDValue Op2 = getValue(I.getArgOperand(1));
7106 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7107 return;
7108 }
7109 case Intrinsic::umax: {
7110 SDValue Op1 = getValue(I.getArgOperand(0));
7111 SDValue Op2 = getValue(I.getArgOperand(1));
7112 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7113 return;
7114 }
7115 case Intrinsic::umin: {
7116 SDValue Op1 = getValue(I.getArgOperand(0));
7117 SDValue Op2 = getValue(I.getArgOperand(1));
7118 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7119 return;
7120 }
7121 case Intrinsic::abs: {
7122 // TODO: Preserve "int min is poison" arg in SDAG?
7123 SDValue Op1 = getValue(I.getArgOperand(0));
7124 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
7125 return;
7126 }
7127 case Intrinsic::stacksave: {
7128 SDValue Op = getRoot();
7129 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7130 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
7131 setValue(&I, Res);
7132 DAG.setRoot(Res.getValue(1));
7133 return;
7134 }
7135 case Intrinsic::stackrestore:
7136 Res = getValue(I.getArgOperand(0));
7137 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7138 return;
7139 case Intrinsic::get_dynamic_area_offset: {
7140 SDValue Op = getRoot();
7141 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7142 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7143 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
7144 // target.
7145 if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
7146 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
7147 " intrinsic!");
7149 Op);
7150 DAG.setRoot(Op);
7151 setValue(&I, Res);
7152 return;
7153 }
7154 case Intrinsic::stackguard: {
7156 const Module &M = *MF.getFunction().getParent();
7157 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7158 SDValue Chain = getRoot();
7159 if (TLI.useLoadStackGuardNode()) {
7160 Res = getLoadStackGuard(DAG, sdl, Chain);
7161 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7162 } else {
7163 const Value *Global = TLI.getSDagStackGuard(M);
7165 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7168 }
7169 if (TLI.useStackGuardXorFP())
7170 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
7171 DAG.setRoot(Chain);
7172 setValue(&I, Res);
7173 return;
7174 }
7175 case Intrinsic::stackprotector: {
7176 // Emit code into the DAG to store the stack guard onto the stack.
7178 MachineFrameInfo &MFI = MF.getFrameInfo();
7179 SDValue Src, Chain = getRoot();
7180
7181 if (TLI.useLoadStackGuardNode())
7182 Src = getLoadStackGuard(DAG, sdl, Chain);
7183 else
7184 Src = getValue(I.getArgOperand(0)); // The guard's value.
7185
7186 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7187
7188 int FI = FuncInfo.StaticAllocaMap[Slot];
7189 MFI.setStackProtectorIndex(FI);
7190 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7191
7192 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7193
7194 // Store the stack protector onto the stack.
7195 Res = DAG.getStore(
7196 Chain, sdl, Src, FIN,
7199 setValue(&I, Res);
7200 DAG.setRoot(Res);
7201 return;
7202 }
7203 case Intrinsic::objectsize:
7204 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7205
7206 case Intrinsic::is_constant:
7207 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7208
7209 case Intrinsic::annotation:
7210 case Intrinsic::ptr_annotation:
7211 case Intrinsic::launder_invariant_group:
7212 case Intrinsic::strip_invariant_group:
7213 // Drop the intrinsic, but forward the value
7214 setValue(&I, getValue(I.getOperand(0)));
7215 return;
7216
7217 case Intrinsic::assume:
7218 case Intrinsic::experimental_noalias_scope_decl:
7219 case Intrinsic::var_annotation:
7220 case Intrinsic::sideeffect:
7221 // Discard annotate attributes, noalias scope declarations, assumptions, and
7222 // artificial side-effects.
7223 return;
7224
7225 case Intrinsic::codeview_annotation: {
7226 // Emit a label associated with this metadata.
7228 MCSymbol *Label =
7229 MF.getMMI().getContext().createTempSymbol("annotation", true);
7230 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7231 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7232 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7233 DAG.setRoot(Res);
7234 return;
7235 }
7236
7237 case Intrinsic::init_trampoline: {
7238 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7239
7240 SDValue Ops[6];
7241 Ops[0] = getRoot();
7242 Ops[1] = getValue(I.getArgOperand(0));
7243 Ops[2] = getValue(I.getArgOperand(1));
7244 Ops[3] = getValue(I.getArgOperand(2));
7245 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7246 Ops[5] = DAG.getSrcValue(F);
7247
7248 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7249
7250 DAG.setRoot(Res);
7251 return;
7252 }
7253 case Intrinsic::adjust_trampoline:
7256 getValue(I.getArgOperand(0))));
7257 return;
7258 case Intrinsic::gcroot: {
7260 "only valid in functions with gc specified, enforced by Verifier");
7261 assert(GFI && "implied by previous");
7262 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7263 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7264
7265 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7266 GFI->addStackRoot(FI->getIndex(), TypeMap);
7267 return;
7268 }
7269 case Intrinsic::gcread:
7270 case Intrinsic::gcwrite:
7271 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7272 case Intrinsic::get_rounding:
7273 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7274 setValue(&I, Res);
7275 DAG.setRoot(Res.getValue(1));
7276 return;
7277
7278 case Intrinsic::expect:
7279 // Just replace __builtin_expect(exp, c) with EXP.
7280 setValue(&I, getValue(I.getArgOperand(0)));
7281 return;
7282
7283 case Intrinsic::ubsantrap:
7284 case Intrinsic::debugtrap:
7285 case Intrinsic::trap: {
7286 StringRef TrapFuncName =
7287 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
7288 if (TrapFuncName.empty()) {
7289 switch (Intrinsic) {
7290 case Intrinsic::trap:
7291 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
7292 break;
7293 case Intrinsic::debugtrap:
7294 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
7295 break;
7296 case Intrinsic::ubsantrap:
7298 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
7300 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
7301 MVT::i32)));
7302 break;
7303 default: llvm_unreachable("unknown trap intrinsic");
7304 }
7305 return;
7306 }
7308 if (Intrinsic == Intrinsic::ubsantrap) {
7310 Args[0].Val = I.getArgOperand(0);
7311 Args[0].Node = getValue(Args[0].Val);
7312 Args[0].Ty = Args[0].Val->getType();
7313 }
7314
7316 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7317 CallingConv::C, I.getType(),
7318 DAG.getExternalSymbol(TrapFuncName.data(),
7320 std::move(Args));
7321
7322 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7323 DAG.setRoot(Result.second);
7324 return;
7325 }
7326
7327 case Intrinsic::uadd_with_overflow:
7328 case Intrinsic::sadd_with_overflow:
7329 case Intrinsic::usub_with_overflow:
7330 case Intrinsic::ssub_with_overflow:
7331 case Intrinsic::umul_with_overflow:
7332 case Intrinsic::smul_with_overflow: {
7334 switch (Intrinsic) {
7335 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7336 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7337 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7338 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7339 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7340 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7341 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7342 }
7343 SDValue Op1 = getValue(I.getArgOperand(0));
7344 SDValue Op2 = getValue(I.getArgOperand(1));
7345
7346 EVT ResultVT = Op1.getValueType();
7347 EVT OverflowVT = MVT::i1;
7348 if (ResultVT.isVector())
7349 OverflowVT = EVT::getVectorVT(
7350 *Context, OverflowVT, ResultVT.getVectorElementCount());
7351
7352 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
7353 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
7354 return;
7355 }
7356 case Intrinsic::prefetch: {
7357 SDValue Ops[5];
7358 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7360 Ops[0] = DAG.getRoot();
7361 Ops[1] = getValue(I.getArgOperand(0));
7362 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7363 MVT::i32);
7364 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
7365 MVT::i32);
7366 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
7367 MVT::i32);
7369 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
7370 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
7371 /* align */ std::nullopt, Flags);
7372
7373 // Chain the prefetch in parallel with any pending loads, to stay out of
7374 // the way of later optimizations.
7375 PendingLoads.push_back(Result);
7376 Result = getRoot();
7377 DAG.setRoot(Result);
7378 return;
7379 }
7380 case Intrinsic::lifetime_start:
7381 case Intrinsic::lifetime_end: {
7382 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7383 // Stack coloring is not enabled in O0, discard region information.
7385 return;
7386
7387 const int64_t ObjectSize =
7388 cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
7389 Value *const ObjectPtr = I.getArgOperand(1);
7391 getUnderlyingObjects(ObjectPtr, Allocas);
7392
7393 for (const Value *Alloca : Allocas) {
7394 const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
7395
7396 // Could not find an Alloca.
7397 if (!LifetimeObject)
7398 continue;
7399
7400 // First check that the Alloca is static, otherwise it won't have a
7401 // valid frame index.
7402 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
7403 if (SI == FuncInfo.StaticAllocaMap.end())
7404 return;
7405
7406 const int FrameIndex = SI->second;
7407 int64_t Offset;
7409 ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
7410 Offset = -1; // Cannot determine offset from alloca to lifetime object.
7411 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
7412 Offset);
7413 DAG.setRoot(Res);
7414 }
7415 return;
7416 }
7417 case Intrinsic::pseudoprobe: {
7418 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
7419 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7420 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
7421 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
7422 DAG.setRoot(Res);
7423 return;
7424 }
7425 case Intrinsic::invariant_start:
7426 // Discard region information.
7427 setValue(&I,
7428 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
7429 return;
7430 case Intrinsic::invariant_end:
7431 // Discard region information.
7432 return;
7433 case Intrinsic::clear_cache:
7434 /// FunctionName may be null.
7435 if (const char *FunctionName = TLI.getClearCacheBuiltinName())
7436 lowerCallToExternalSymbol(I, FunctionName);
7437 return;
7438 case Intrinsic::donothing:
7439 case Intrinsic::seh_try_begin:
7440 case Intrinsic::seh_scope_begin:
7441 case Intrinsic::seh_try_end:
7442 case Intrinsic::seh_scope_end:
7443 // ignore
7444 return;
7445 case Intrinsic::experimental_stackmap:
7446 visitStackmap(I);
7447 return;
7448 case Intrinsic::experimental_patchpoint_void:
7449 case Intrinsic::experimental_patchpoint_i64:
7450 visitPatchpoint(I);
7451 return;
7452 case Intrinsic::experimental_gc_statepoint:
7453 LowerStatepoint(cast<GCStatepointInst>(I));
7454 return;
7455 case Intrinsic::experimental_gc_result:
7456 visitGCResult(cast<GCResultInst>(I));
7457 return;
7458 case Intrinsic::experimental_gc_relocate:
7459 visitGCRelocate(cast<GCRelocateInst>(I));
7460 return;
7461 case Intrinsic::instrprof_cover:
7462 llvm_unreachable("instrprof failed to lower a cover");
7463 case Intrinsic::instrprof_increment:
7464 llvm_unreachable("instrprof failed to lower an increment");
7465 case Intrinsic::instrprof_timestamp:
7466 llvm_unreachable("instrprof failed to lower a timestamp");
7467 case Intrinsic::instrprof_value_profile:
7468 llvm_unreachable("instrprof failed to lower a value profiling call");
7469 case Intrinsic::instrprof_mcdc_parameters:
7470 llvm_unreachable("instrprof failed to lower mcdc parameters");
7471 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7472 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7473 case Intrinsic::instrprof_mcdc_condbitmap_update:
7474 llvm_unreachable("instrprof failed to lower an mcdc condbitmap update");
7475 case Intrinsic::localescape: {
7478
7479 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7480 // is the same on all targets.
7481 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7482 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
7483 if (isa<ConstantPointerNull>(Arg))
7484 continue; // Skip null pointers. They represent a hole in index space.
7485 AllocaInst *Slot = cast<AllocaInst>(Arg);
7486 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7487 "can only escape static allocas");
7488 int FI = FuncInfo.StaticAllocaMap[Slot];
7489 MCSymbol *FrameAllocSym =
7493 TII->get(TargetOpcode::LOCAL_ESCAPE))
7494 .addSym(FrameAllocSym)
7495 .addFrameIndex(FI);
7496 }
7497
7498 return;
7499 }
7500
7501 case Intrinsic::localrecover: {
7502 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7504
7505 // Get the symbol that defines the frame offset.
7506 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
7507 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
7508 unsigned IdxVal =
7509 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
7510 MCSymbol *FrameAllocSym =
7513
7514 Value *FP = I.getArgOperand(1);
7515 SDValue FPVal = getValue(FP);
7516 EVT PtrVT = FPVal.getValueType();
7517
7518 // Create a MCSymbol for the label to avoid any target lowering
7519 // that would make this PC relative.
7520 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
7521 SDValue OffsetVal =
7522 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
7523
7524 // Add the offset to the FP.
7525 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
7526 setValue(&I, Add);
7527
7528 return;
7529 }
7530
7531 case Intrinsic::eh_exceptionpointer:
7532 case Intrinsic::eh_exceptioncode: {
7533 // Get the exception pointer vreg, copy from it, and resize it to fit.
7534 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7535 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7536 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7537 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7538 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7539 if (Intrinsic == Intrinsic::eh_exceptioncode)
7540 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7541 setValue(&I, N);
7542 return;
7543 }
7544 case Intrinsic::xray_customevent: {
7545 // Here we want to make sure that the intrinsic behaves as if it has a
7546 // specific calling convention.
7547 const auto &Triple = DAG.getTarget().getTargetTriple();
7548 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7549 return;
7550
7552
7553 // We want to say that we always want the arguments in registers.
7554 SDValue LogEntryVal = getValue(I.getArgOperand(0));
7555 SDValue StrSizeVal = getValue(I.getArgOperand(1));
7556 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7557 SDValue Chain = getRoot();
7558 Ops.push_back(LogEntryVal);
7559 Ops.push_back(StrSizeVal);
7560 Ops.push_back(Chain);
7561
7562 // We need to enforce the calling convention for the callsite, so that
7563 // argument ordering is enforced correctly, and that register allocation can
7564 // see that some registers may be assumed clobbered and have to preserve
7565 // them across calls to the intrinsic.
7566 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7567 sdl, NodeTys, Ops);
7568 SDValue patchableNode = SDValue(MN, 0);
7569 DAG.setRoot(patchableNode);
7570 setValue(&I, patchableNode);
7571 return;
7572 }
7573 case Intrinsic::xray_typedevent: {
7574 // Here we want to make sure that the intrinsic behaves as if it has a
7575 // specific calling convention.
7576 const auto &Triple = DAG.getTarget().getTargetTriple();
7577 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7578 return;
7579
7581
7582 // We want to say that we always want the arguments in registers.
7583 // It's unclear to me how manipulating the selection DAG here forces callers
7584 // to provide arguments in registers instead of on the stack.
7585 SDValue LogTypeId = getValue(I.getArgOperand(0));
7586 SDValue LogEntryVal = getValue(I.getArgOperand(1));
7587 SDValue StrSizeVal = getValue(I.getArgOperand(2));
7588 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7589 SDValue Chain = getRoot();
7590 Ops.push_back(LogTypeId);
7591 Ops.push_back(LogEntryVal);
7592 Ops.push_back(StrSizeVal);
7593 Ops.push_back(Chain);
7594
7595 // We need to enforce the calling convention for the callsite, so that
7596 // argument ordering is enforced correctly, and that register allocation can
7597 // see that some registers may be assumed clobbered and have to preserve
7598 // them across calls to the intrinsic.
7600 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7601 SDValue patchableNode = SDValue(MN, 0);
7602 DAG.setRoot(patchableNode);
7603 setValue(&I, patchableNode);
7604 return;
7605 }
7606 case Intrinsic::experimental_deoptimize:
7608 return;
7609 case Intrinsic::experimental_stepvector:
7610 visitStepVector(I);
7611 return;
7612 case Intrinsic::vector_reduce_fadd:
7613 case Intrinsic::vector_reduce_fmul:
7614 case Intrinsic::vector_reduce_add:
7615 case Intrinsic::vector_reduce_mul:
7616 case Intrinsic::vector_reduce_and:
7617 case Intrinsic::vector_reduce_or:
7618 case Intrinsic::vector_reduce_xor:
7619 case Intrinsic::vector_reduce_smax:
7620 case Intrinsic::vector_reduce_smin:
7621 case Intrinsic::vector_reduce_umax:
7622 case Intrinsic::vector_reduce_umin:
7623 case Intrinsic::vector_reduce_fmax:
7624 case Intrinsic::vector_reduce_fmin:
7625 case Intrinsic::vector_reduce_fmaximum:
7626 case Intrinsic::vector_reduce_fminimum:
7627 visitVectorReduce(I, Intrinsic);
7628 return;
7629
7630 case Intrinsic::icall_branch_funnel: {
7632 Ops.push_back(getValue(I.getArgOperand(0)));
7633
7634 int64_t Offset;
7635 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7636 I.getArgOperand(1), Offset, DAG.getDataLayout()));
7637 if (!Base)
7639 "llvm.icall.branch.funnel operand must be a GlobalValue");
7640 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7641
7642 struct BranchFunnelTarget {
7643 int64_t Offset;
7645 };
7647
7648 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7649 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7650 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7651 if (ElemBase != Base)
7652 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7653 "to the same GlobalValue");
7654
7655 SDValue Val = getValue(I.getArgOperand(Op + 1));
7656 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7657 if (!GA)
7659 "llvm.icall.branch.funnel operand must be a GlobalValue");
7661 GA->getGlobal(), sdl, Val.getValueType(),
7662 GA->getOffset())});
7663 }
7664 llvm::sort(Targets,
7665 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7666 return T1.Offset < T2.Offset;
7667 });
7668
7669 for (auto &T : Targets) {
7670 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7671 Ops.push_back(T.Target);
7672 }
7673
7674 Ops.push_back(DAG.getRoot()); // Chain
7675 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7676 MVT::Other, Ops),
7677 0);
7678 DAG.setRoot(N);
7679 setValue(&I, N);
7680 HasTailCall = true;
7681 return;
7682 }
7683
7684 case Intrinsic::wasm_landingpad_index:
7685 // Information this intrinsic contained has been transferred to
7686 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7687 // delete it now.
7688 return;
7689
7690 case Intrinsic::aarch64_settag:
7691 case Intrinsic::aarch64_settag_zero: {
7693 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7695 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7696 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7697 ZeroMemory);
7698 DAG.setRoot(Val);
7699 setValue(&I, Val);
7700 return;
7701 }
7702 case Intrinsic::amdgcn_cs_chain: {
7703 assert(I.arg_size() == 5 && "Additional args not supported yet");
7704 assert(cast<ConstantInt>(I.getOperand(4))->isZero() &&
7705 "Non-zero flags not supported yet");
7706
7707 // At this point we don't care if it's amdgpu_cs_chain or
7708 // amdgpu_cs_chain_preserve.
7710
7711 Type *RetTy = I.getType();
7712 assert(RetTy->isVoidTy() && "Should not return");
7713
7714 SDValue Callee = getValue(I.getOperand(0));
7715
7716 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
7717 // We'll also tack the value of the EXEC mask at the end.
7719 Args.reserve(3);
7720
7721 for (unsigned Idx : {2, 3, 1}) {
7723 Arg.Node = getValue(I.getOperand(Idx));
7724 Arg.Ty = I.getOperand(Idx)->getType();
7725 Arg.setAttributes(&I, Idx);
7726 Args.push_back(Arg);
7727 }
7728
7729 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
7730 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
7731 Args[2].IsInReg = true; // EXEC should be inreg
7732
7734 CLI.setDebugLoc(getCurSDLoc())
7735 .setChain(getRoot())
7736 .setCallee(CC, RetTy, Callee, std::move(Args))
7737 .setNoReturn(true)
7738 .setTailCall(true)
7739 .setConvergent(I.isConvergent());
7740 CLI.CB = &I;
7741 std::pair<SDValue, SDValue> Result =
7742 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
7743 (void)Result;
7744 assert(!Result.first.getNode() && !Result.second.getNode() &&
7745 "Should've lowered as tail call");
7746
7747 HasTailCall = true;
7748 return;
7749 }
7750 case Intrinsic::ptrmask: {
7751 SDValue Ptr = getValue(I.getOperand(0));
7752 SDValue Mask = getValue(I.getOperand(1));
7753
7754 EVT PtrVT = Ptr.getValueType();
7755 assert(PtrVT == Mask.getValueType() &&
7756 "Pointers with different index type are not supported by SDAG");
7757 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
7758 return;
7759 }
7760 case Intrinsic::threadlocal_address: {
7761 setValue(&I, getValue(I.getOperand(0)));
7762 return;
7763 }
7764 case Intrinsic::get_active_lane_mask: {
7765 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7766 SDValue Index = getValue(I.getOperand(0));
7767 EVT ElementVT = Index.getValueType();
7768
7769 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7770 visitTargetIntrinsic(I, Intrinsic);
7771 return;
7772 }
7773
7774 SDValue TripCount = getValue(I.getOperand(1));
7775 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
7776 CCVT.getVectorElementCount());
7777
7778 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
7779 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
7780 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7781 SDValue VectorInduction = DAG.getNode(
7782 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7783 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7784 VectorTripCount, ISD::CondCode::SETULT);
7785 setValue(&I, SetCC);
7786 return;
7787 }
7788 case Intrinsic::experimental_get_vector_length: {
7789 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
7790 "Expected positive VF");
7791 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
7792 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
7793
7794 SDValue Count = getValue(I.getOperand(0));
7795 EVT CountVT = Count.getValueType();
7796
7797 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
7798 visitTargetIntrinsic(I, Intrinsic);
7799 return;
7800 }
7801
7802 // Expand to a umin between the trip count and the maximum elements the type
7803 // can hold.
7804 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7805
7806 // Extend the trip count to at least the result VT.
7807 if (CountVT.bitsLT(VT)) {
7808 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
7809 CountVT = VT;
7810 }
7811
7812 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
7813 ElementCount::get(VF, IsScalable));
7814
7815 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
7816 // Clip to the result type if needed.
7817 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
7818
7819 setValue(&I, Trunc);
7820 return;
7821 }
7822 case Intrinsic::experimental_cttz_elts: {
7823 auto DL = getCurSDLoc();
7824 SDValue Op = getValue(I.getOperand(0));
7825 EVT OpVT = Op.getValueType();
7826
7827 if (!TLI.shouldExpandCttzElements(OpVT)) {
7828 visitTargetIntrinsic(I, Intrinsic);
7829 return;
7830 }
7831
7832 if (OpVT.getScalarType() != MVT::i1) {
7833 // Compare the input vector elements to zero & use to count trailing zeros
7834 SDValue AllZero = DAG.getConstant(0, DL, OpVT);
7835 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
7836 OpVT.getVectorElementCount());
7837 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
7838 }
7839
7840 // Find the smallest "sensible" element type to use for the expansion.
7841 ConstantRange CR(
7843 if (OpVT.isScalableVT())
7844 CR = CR.umul_sat(getVScaleRange(I.getCaller(), 64));
7845
7846 // If the zero-is-poison flag is set, we can assume the upper limit
7847 // of the result is VF-1.
7848 if (!cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero())
7849 CR = CR.subtract(APInt(64, 1));
7850
7851 unsigned EltWidth = I.getType()->getScalarSizeInBits();
7852 EltWidth = std::min(EltWidth, (unsigned)CR.getActiveBits());
7853 EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);
7854
7855 MVT NewEltTy = MVT::getIntegerVT(EltWidth);
7856
7857 // Create the new vector type & get the vector length
7858 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,
7859 OpVT.getVectorElementCount());
7860
7861 SDValue VL =
7862 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());
7863
7864 SDValue StepVec = DAG.getStepVector(DL, NewVT);
7865 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
7866 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
7868 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
7870 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
7871
7872 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7874
7875 setValue(&I, Ret);
7876 return;
7877 }
7878 case Intrinsic::vector_insert: {
7879 SDValue Vec = getValue(I.getOperand(0));
7880 SDValue SubVec = getValue(I.getOperand(1));
7881 SDValue Index = getValue(I.getOperand(2));
7882
7883 // The intrinsic's index type is i64, but the SDNode requires an index type
7884 // suitable for the target. Convert the index as required.
7885 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7886 if (Index.getValueType() != VectorIdxTy)
7887 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
7888
7889 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7890 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
7891 Index));
7892 return;
7893 }
7894 case Intrinsic::vector_extract: {
7895 SDValue Vec = getValue(I.getOperand(0));
7896 SDValue Index = getValue(I.getOperand(1));
7897 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7898
7899 // The intrinsic's index type is i64, but the SDNode requires an index type
7900 // suitable for the target. Convert the index as required.
7901 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7902 if (Index.getValueType() != VectorIdxTy)
7903 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
7904
7905 setValue(&I,
7906 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
7907 return;
7908 }
7909 case Intrinsic::experimental_vector_reverse:
7910 visitVectorReverse(I);
7911 return;
7912 case Intrinsic::experimental_vector_splice:
7913 visitVectorSplice(I);
7914 return;
7915 case Intrinsic::callbr_landingpad:
7916 visitCallBrLandingPad(I);
7917 return;
7918 case Intrinsic::experimental_vector_interleave2:
7919 visitVectorInterleave(I);
7920 return;
7921 case Intrinsic::experimental_vector_deinterleave2:
7922 visitVectorDeinterleave(I);
7923 return;
7924 case Intrinsic::experimental_convergence_anchor:
7925 case Intrinsic::experimental_convergence_entry:
7926 case Intrinsic::experimental_convergence_loop:
7927 visitConvergenceControl(I, Intrinsic);
7928 }
7929}
7930
7931void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
7932 const ConstrainedFPIntrinsic &FPI) {
7933 SDLoc sdl = getCurSDLoc();
7934
7935 // We do not need to serialize constrained FP intrinsics against
7936 // each other or against (nonvolatile) loads, so they can be
7937 // chained like loads.
7938 SDValue Chain = DAG.getRoot();
7940 Opers.push_back(Chain);
7941 if (FPI.isUnaryOp()) {
7942 Opers.push_back(getValue(FPI.getArgOperand(0)));
7943 } else if (FPI.isTernaryOp()) {
7944 Opers.push_back(getValue(FPI.getArgOperand(0)));
7945 Opers.push_back(getValue(FPI.getArgOperand(1)));
7946 Opers.push_back(getValue(FPI.getArgOperand(2)));
7947 } else {
7948 Opers.push_back(getValue(FPI.getArgOperand(0)));
7949 Opers.push_back(getValue(FPI.getArgOperand(1)));
7950 }
7951
7952 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
7953 assert(Result.getNode()->getNumValues() == 2);
7954
7955 // Push node to the appropriate list so that future instructions can be
7956 // chained up correctly.
7957 SDValue OutChain = Result.getValue(1);
7958 switch (EB) {
7960 // The only reason why ebIgnore nodes still need to be chained is that
7961 // they might depend on the current rounding mode, and therefore must
7962 // not be moved across instruction that may change that mode.
7963 [[fallthrough]];
7965 // These must not be moved across calls or instructions that may change
7966 // floating-point exception masks.
7967 PendingConstrainedFP.push_back(OutChain);
7968 break;
7970 // These must not be moved across calls or instructions that may change
7971 // floating-point exception masks or read floating-point exception flags.
7972 // In addition, they cannot be optimized out even if unused.
7973 PendingConstrainedFPStrict.push_back(OutChain);
7974 break;
7975 }
7976 };
7977
7979 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
7980 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
7982
7985 Flags.setNoFPExcept(true);
7986
7987 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
7988 Flags.copyFMF(*FPOp);
7989
7990 unsigned Opcode;
7991 switch (FPI.getIntrinsicID()) {
7992 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7993#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
7994 case Intrinsic::INTRINSIC: \
7995 Opcode = ISD::STRICT_##DAGN; \
7996 break;
7997#include "llvm/IR/ConstrainedOps.def"
7998 case Intrinsic::experimental_constrained_fmuladd: {
7999 Opcode = ISD::STRICT_FMA;
8000 // Break fmuladd into fmul and fadd.
8003 Opers.pop_back();
8004 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8005 pushOutChain(Mul, EB);
8006 Opcode = ISD::STRICT_FADD;
8007 Opers.clear();
8008 Opers.push_back(Mul.getValue(1));
8009 Opers.push_back(Mul.getValue(0));
8010 Opers.push_back(getValue(FPI.getArgOperand(2)));
8011 }
8012 break;
8013 }
8014 }
8015
8016 // A few strict DAG nodes carry additional operands that are not
8017 // set up by the default code above.
8018 switch (Opcode) {
8019 default: break;
8021 Opers.push_back(
8023 break;
8024 case ISD::STRICT_FSETCC:
8025 case ISD::STRICT_FSETCCS: {
8026 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8027 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8028 if (TM.Options.NoNaNsFPMath)
8029 Condition = getFCmpCodeWithoutNaN(Condition);
8030 Opers.push_back(DAG.getCondCode(Condition));
8031 break;
8032 }
8033 }
8034
8035 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8036 pushOutChain(Result, EB);
8037
8038 SDValue FPResult = Result.getValue(0);
8039 setValue(&FPI, FPResult);
8040}
8041
8042static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8043 std::optional<unsigned> ResOPC;
8044 switch (VPIntrin.getIntrinsicID()) {
8045 case Intrinsic::vp_ctlz: {
8046 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8047 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8048 break;
8049 }
8050 case Intrinsic::vp_cttz: {
8051 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8052 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8053 break;
8054 }
8055#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8056 case Intrinsic::VPID: \
8057 ResOPC = ISD::VPSD; \
8058 break;
8059#include "llvm/IR/VPIntrinsics.def"
8060 }
8061
8062 if (!ResOPC)
8064 "Inconsistency: no SDNode available for this VPIntrinsic!");
8065
8066 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8067 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8068 if (VPIntrin.getFastMathFlags().allowReassoc())
8069 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8070 : ISD::VP_REDUCE_FMUL;
8071 }
8072
8073 return *ResOPC;
8074}
8075
8076void SelectionDAGBuilder::visitVPLoad(
8077 const VPIntrinsic &VPIntrin, EVT VT,
8078 const SmallVectorImpl<SDValue> &OpValues) {
8079 SDLoc DL = getCurSDLoc();
8080 Value *PtrOperand = VPIntrin.getArgOperand(0);
8081 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8082 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8083 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8084 SDValue LD;
8085 // Do not serialize variable-length loads of constant memory with
8086 // anything.
8087 if (!Alignment)
8088 Alignment = DAG.getEVTAlign(VT);
8089 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8090 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8091 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8094 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8095 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
8096 MMO, false /*IsExpanding */);
8097 if (AddToChain)
8098 PendingLoads.push_back(LD.getValue(1));
8099 setValue(&VPIntrin, LD);
8100}
8101
8102void SelectionDAGBuilder::visitVPGather(
8103 const VPIntrinsic &VPIntrin, EVT VT,
8104 const SmallVectorImpl<SDValue> &OpValues) {
8105 SDLoc DL = getCurSDLoc();
8107 Value *PtrOperand = VPIntrin.getArgOperand(0);
8108 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8109 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8110 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8111 SDValue LD;
8112 if (!Alignment)
8113 Alignment = DAG.getEVTAlign(VT.getScalarType());
8114 unsigned AS =
8115 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8118 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8119 SDValue Base, Index, Scale;
8120 ISD::MemIndexType IndexType;
8121 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8122 this, VPIntrin.getParent(),
8123 VT.getScalarStoreSize());
8124 if (!UniformBase) {
8126 Index = getValue(PtrOperand);
8127 IndexType = ISD::SIGNED_SCALED;
8129 }
8130 EVT IdxVT = Index.getValueType();
8131 EVT EltTy = IdxVT.getVectorElementType();
8132 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8133 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8134 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8135 }
8136 LD = DAG.getGatherVP(
8137 DAG.getVTList(VT, MVT::Other), VT, DL,
8138 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8139 IndexType);
8140 PendingLoads.push_back(LD.getValue(1));
8141 setValue(&VPIntrin, LD);
8142}
8143
8144void SelectionDAGBuilder::visitVPStore(
8145 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8146 SDLoc DL = getCurSDLoc();
8147 Value *PtrOperand = VPIntrin.getArgOperand(1);
8148 EVT VT = OpValues[0].getValueType();
8149 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8150 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8151 SDValue ST;
8152 if (!Alignment)
8153 Alignment = DAG.getEVTAlign(VT);
8154 SDValue Ptr = OpValues[1];
8155 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
8158 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8159 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
8160 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
8161 /* IsTruncating */ false, /*IsCompressing*/ false);
8162 DAG.setRoot(ST);
8163 setValue(&VPIntrin, ST);
8164}
8165
8166void SelectionDAGBuilder::visitVPScatter(
8167 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8168 SDLoc DL = getCurSDLoc();
8170 Value *PtrOperand = VPIntrin.getArgOperand(1);
8171 EVT VT = OpValues[0].getValueType();
8172 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8173 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8174 SDValue ST;
8175 if (!Alignment)
8176 Alignment = DAG.getEVTAlign(VT.getScalarType());
8177 unsigned AS =
8178 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8181 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8182 SDValue Base, Index, Scale;
8183 ISD::MemIndexType IndexType;
8184 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8185 this, VPIntrin.getParent(),
8186 VT.getScalarStoreSize());
8187 if (!UniformBase) {
8189 Index = getValue(PtrOperand);
8190 IndexType = ISD::SIGNED_SCALED;
8191 Scale =
8193 }
8194 EVT IdxVT = Index.getValueType();
8195 EVT EltTy = IdxVT.getVectorElementType();
8196 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8197 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8198 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8199 }
8200 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
8201 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8202 OpValues[2], OpValues[3]},
8203 MMO, IndexType);
8204 DAG.setRoot(ST);
8205 setValue(&VPIntrin, ST);
8206}
8207
8208void SelectionDAGBuilder::visitVPStridedLoad(
8209 const VPIntrinsic &VPIntrin, EVT VT,
8210 const SmallVectorImpl<SDValue> &OpValues) {
8211 SDLoc DL = getCurSDLoc();
8212 Value *PtrOperand = VPIntrin.getArgOperand(0);
8213 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8214 if (!Alignment)
8215 Alignment = DAG.getEVTAlign(VT.getScalarType());
8216 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8217 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8218 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8219 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8220 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8221 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8224 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8225
8226 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
8227 OpValues[2], OpValues[3], MMO,
8228 false /*IsExpanding*/);
8229
8230 if (AddToChain)
8231 PendingLoads.push_back(LD.getValue(1));
8232 setValue(&VPIntrin, LD);
8233}
8234
8235void SelectionDAGBuilder::visitVPStridedStore(
8236 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8237 SDLoc DL = getCurSDLoc();
8238 Value *PtrOperand = VPIntrin.getArgOperand(1);
8239 EVT VT = OpValues[0].getValueType();
8240 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8241 if (!Alignment)
8242 Alignment = DAG.getEVTAlign(VT.getScalarType());
8243 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8244 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8247 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8248
8250 getMemoryRoot(), DL, OpValues[0], OpValues[1],
8251 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
8252 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
8253 /*IsCompressing*/ false);
8254
8255 DAG.setRoot(ST);
8256 setValue(&VPIntrin, ST);
8257}
8258
8259void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8261 SDLoc DL = getCurSDLoc();
8262
8263 ISD::CondCode Condition;
8265 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
8266 if (IsFP) {
8267 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
8268 // flags, but calls that don't return floating-point types can't be
8269 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
8270 Condition = getFCmpCondCode(CondCode);
8271 if (TM.Options.NoNaNsFPMath)
8272 Condition = getFCmpCodeWithoutNaN(Condition);
8273 } else {
8274 Condition = getICmpCondCode(CondCode);
8275 }
8276
8277 SDValue Op1 = getValue(VPIntrin.getOperand(0));
8278 SDValue Op2 = getValue(VPIntrin.getOperand(1));
8279 // #2 is the condition code
8280 SDValue MaskOp = getValue(VPIntrin.getOperand(3));
8281 SDValue EVL = getValue(VPIntrin.getOperand(4));
8282 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8283 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8284 "Unexpected target EVL type");
8285 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
8286
8288 VPIntrin.getType());
8289 setValue(&VPIntrin,
8290 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
8291}
8292
8293void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8294 const VPIntrinsic &VPIntrin) {
8295 SDLoc DL = getCurSDLoc();
8296 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8297
8298 auto IID = VPIntrin.getIntrinsicID();
8299
8300 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
8301 return visitVPCmp(*CmpI);
8302
8303 SmallVector<EVT, 4> ValueVTs;
8305 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
8306 SDVTList VTs = DAG.getVTList(ValueVTs);
8307
8308 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
8309
8310 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8311 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8312 "Unexpected target EVL type");
8313
8314 // Request operands.
8315 SmallVector<SDValue, 7> OpValues;
8316 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8317 auto Op = getValue(VPIntrin.getArgOperand(I));
8318 if (I == EVLParamPos)
8319 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
8320 OpValues.push_back(Op);
8321 }
8322
8323 switch (Opcode) {
8324 default: {
8325 SDNodeFlags SDFlags;
8326 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8327 SDFlags.copyFMF(*FPMO);
8328 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
8329 setValue(&VPIntrin, Result);
8330 break;
8331 }
8332 case ISD::VP_LOAD:
8333 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
8334 break;
8335 case ISD::VP_GATHER:
8336 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
8337 break;
8338 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8339 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
8340 break;
8341 case ISD::VP_STORE:
8342 visitVPStore(VPIntrin, OpValues);
8343 break;
8344 case ISD::VP_SCATTER:
8345 visitVPScatter(VPIntrin, OpValues);
8346 break;
8347 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8348 visitVPStridedStore(VPIntrin, OpValues);
8349 break;
8350 case ISD::VP_FMULADD: {
8351 assert(OpValues.size() == 5 && "Unexpected number of operands");
8352 SDNodeFlags SDFlags;
8353 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8354 SDFlags.copyFMF(*FPMO);
8357 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
8358 } else {
8360 ISD::VP_FMUL, DL, VTs,
8361 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
8362 SDValue Add =
8363 DAG.getNode(ISD::VP_FADD, DL, VTs,
8364 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
8365 setValue(&VPIntrin, Add);
8366 }
8367 break;
8368 }
8369 case ISD::VP_IS_FPCLASS: {
8370 const DataLayout DLayout = DAG.getDataLayout();
8371 EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType());
8372 auto Constant = OpValues[1]->getAsZExtVal();
8374 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT,
8375 {OpValues[0], Check, OpValues[2], OpValues[3]});
8376 setValue(&VPIntrin, V);
8377 return;
8378 }
8379 case ISD::VP_INTTOPTR: {
8380 SDValue N = OpValues[0];
8381 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
8382 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
8383 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8384 OpValues[2]);
8385 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8386 OpValues[2]);
8387 setValue(&VPIntrin, N);
8388 break;
8389 }
8390 case ISD::VP_PTRTOINT: {
8391 SDValue N = OpValues[0];
8393 VPIntrin.getType());
8394 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
8395 VPIntrin.getOperand(0)->getType());
8396 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8397 OpValues[2]);
8398 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8399 OpValues[2]);
8400 setValue(&VPIntrin, N);
8401 break;
8402 }
8403 case ISD::VP_ABS:
8404 case ISD::VP_CTLZ:
8405 case ISD::VP_CTLZ_ZERO_UNDEF:
8406 case ISD::VP_CTTZ:
8407 case ISD::VP_CTTZ_ZERO_UNDEF: {
8408 SDValue Result =
8409 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
8410 setValue(&VPIntrin, Result);
8411 break;
8412 }
8413 }
8414}
8415
8416SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
8417 const BasicBlock *EHPadBB,
8418 MCSymbol *&BeginLabel) {
8420 MachineModuleInfo &MMI = MF.getMMI();
8421
8422 // Insert a label before the invoke call to mark the try range. This can be
8423 // used to detect deletion of the invoke via the MachineModuleInfo.
8424 BeginLabel = MMI.getContext().createTempSymbol();
8425
8426 // For SjLj, keep track of which landing pads go with which invokes
8427 // so as to maintain the ordering of pads in the LSDA.
8428 unsigned CallSiteIndex = MMI.getCurrentCallSite();
8429 if (CallSiteIndex) {
8430 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
8431 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
8432
8433 // Now that the call site is handled, stop tracking it.
8434 MMI.setCurrentCallSite(0);
8435 }
8436
8437 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
8438}
8439
8440SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
8441 const BasicBlock *EHPadBB,
8442 MCSymbol *BeginLabel) {
8443 assert(BeginLabel && "BeginLabel should've been set");
8444
8446 MachineModuleInfo &MMI = MF.getMMI();
8447
8448 // Insert a label at the end of the invoke call to mark the try range. This
8449 // can be used to detect deletion of the invoke via the MachineModuleInfo.
8450 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
8451 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
8452
8453 // Inform MachineModuleInfo of range.
8455 // There is a platform (e.g. wasm) that uses funclet style IR but does not
8456 // actually use outlined funclets and their LSDA info style.
8457 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
8458 assert(II && "II should've been set");
8459 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
8460 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
8461 } else if (!isScopedEHPersonality(Pers)) {
8462 assert(EHPadBB);
8463 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
8464 }
8465
8466 return Chain;
8467}
8468
8469std::pair<SDValue, SDValue>
8471 const BasicBlock *EHPadBB) {
8472 MCSymbol *BeginLabel = nullptr;
8473
8474 if (EHPadBB) {
8475 // Both PendingLoads and PendingExports must be flushed here;
8476 // this call might not return.
8477 (void)getRoot();
8478 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
8479 CLI.setChain(getRoot());
8480 }
8481
8483 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8484
8485 assert((CLI.IsTailCall || Result.second.getNode()) &&
8486 "Non-null chain expected with non-tail call!");
8487 assert((Result.second.getNode() || !Result.first.getNode()) &&
8488 "Null value expected with tail call!");
8489
8490 if (!Result.second.getNode()) {
8491 // As a special case, a null chain means that a tail call has been emitted
8492 // and the DAG root is already updated.
8493 HasTailCall = true;
8494
8495 // Since there's no actual continuation from this block, nothing can be
8496 // relying on us setting vregs for them.
8497 PendingExports.clear();
8498 } else {
8499 DAG.setRoot(Result.second);
8500 }
8501
8502 if (EHPadBB) {
8503 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
8504 BeginLabel));
8505 }
8506
8507 return Result;
8508}
8509
8511 bool isTailCall,
8512 bool isMustTailCall,
8513 const BasicBlock *EHPadBB) {
8514 auto &DL = DAG.getDataLayout();
8515 FunctionType *FTy = CB.getFunctionType();
8516 Type *RetTy = CB.getType();
8517
8519 Args.reserve(CB.arg_size());
8520
8521 const Value *SwiftErrorVal = nullptr;
8523
8524 if (isTailCall) {
8525 // Avoid emitting tail calls in functions with the disable-tail-calls
8526 // attribute.
8527 auto *Caller = CB.getParent()->getParent();
8528 if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
8529 "true" && !isMustTailCall)
8530 isTailCall = false;
8531
8532 // We can't tail call inside a function with a swifterror argument. Lowering
8533 // does not support this yet. It would have to move into the swifterror
8534 // register before the call.
8535 if (TLI.supportSwiftError() &&
8536 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8537 isTailCall = false;
8538 }
8539
8540 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
8542 const Value *V = *I;
8543
8544 // Skip empty types
8545 if (V->getType()->isEmptyTy())
8546 continue;
8547
8548 SDValue ArgNode = getValue(V);
8549 Entry.Node = ArgNode; Entry.Ty = V->getType();
8550
8551 Entry.setAttributes(&CB, I - CB.arg_begin());
8552
8553 // Use swifterror virtual register as input to the call.
8554 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
8555 SwiftErrorVal = V;
8556 // We find the virtual register for the actual swifterror argument.
8557 // Instead of using the Value, we use the virtual register instead.
8558 Entry.Node =
8560 EVT(TLI.getPointerTy(DL)));
8561 }
8562
8563 Args.push_back(Entry);
8564
8565 // If we have an explicit sret argument that is an Instruction, (i.e., it
8566 // might point to function-local memory), we can't meaningfully tail-call.
8567 if (Entry.IsSRet && isa<Instruction>(V))
8568 isTailCall = false;
8569 }
8570
8571 // If call site has a cfguardtarget operand bundle, create and add an
8572 // additional ArgListEntry.
8573 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
8575 Value *V = Bundle->Inputs[0];
8576 SDValue ArgNode = getValue(V);
8577 Entry.Node = ArgNode;
8578 Entry.Ty = V->getType();
8579 Entry.IsCFGuardTarget = true;
8580 Args.push_back(Entry);
8581 }
8582
8583 // Check if target-independent constraints permit a tail call here.
8584 // Target-dependent constraints are checked within TLI->LowerCallTo.
8585 if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
8586 isTailCall = false;
8587
8588 // Disable tail calls if there is an swifterror argument. Targets have not
8589 // been updated to support tail calls.
8590 if (TLI.supportSwiftError() && SwiftErrorVal)
8591 isTailCall = false;
8592
8593 ConstantInt *CFIType = nullptr;
8594 if (CB.isIndirectCall()) {
8595 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
8596 if (!TLI.supportKCFIBundles())
8598 "Target doesn't support calls with kcfi operand bundles.");
8599 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8600 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8601 }
8602 }
8603
8604 SDValue ConvControlToken;
8605 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8606 auto *Token = Bundle->Inputs[0].get();
8607 ConvControlToken = getValue(Token);
8608 } else {
8609 ConvControlToken = DAG.getUNDEF(MVT::Untyped);
8610 }
8611
8614 .setChain(getRoot())
8615 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
8616 .setTailCall(isTailCall)
8620 .setCFIType(CFIType)
8621 .setConvergenceControlToken(ConvControlToken);
8622 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8623
8624 if (Result.first.getNode()) {
8625 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
8626 setValue(&CB, Result.first);
8627 }
8628
8629 // The last element of CLI.InVals has the SDValue for swifterror return.
8630 // Here we copy it to a virtual register and update SwiftErrorMap for
8631 // book-keeping.
8632 if (SwiftErrorVal && TLI.supportSwiftError()) {
8633 // Get the last element of InVals.
8634 SDValue Src = CLI.InVals.back();
8635 Register VReg =
8636 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
8637 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
8638 DAG.setRoot(CopyNode);
8639 }
8640}
8641
8642static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
8643 SelectionDAGBuilder &Builder) {
8644 // Check to see if this load can be trivially constant folded, e.g. if the
8645 // input is from a string literal.
8646 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
8647 // Cast pointer to the type we really want to load.
8648 Type *LoadTy =
8649 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
8650 if (LoadVT.isVector())
8651 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
8652
8653 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
8654 PointerType::getUnqual(LoadTy));
8655
8656 if (const Constant *LoadCst =
8657 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
8658 LoadTy, Builder.DAG.getDataLayout()))
8659 return Builder.getValue(LoadCst);
8660 }
8661
8662 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
8663 // still constant memory, the input chain can be the entry node.
8664 SDValue Root;
8665 bool ConstantMemory = false;
8666
8667 // Do not serialize (non-volatile) loads of constant memory with anything.
8668 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
8669 Root = Builder.DAG.getEntryNode();
8670 ConstantMemory = true;
8671 } else {
8672 // Do not serialize non-volatile loads against each other.
8673 Root = Builder.DAG.getRoot();
8674 }
8675
8676 SDValue Ptr = Builder.getValue(PtrVal);
8677 SDValue LoadVal =
8678 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
8679 MachinePointerInfo(PtrVal), Align(1));
8680
8681 if (!ConstantMemory)
8682 Builder.PendingLoads.push_back(LoadVal.getValue(1));
8683 return LoadVal;
8684}
8685
8686/// Record the value for an instruction that produces an integer result,
8687/// converting the type where necessary.
8688void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
8689 SDValue Value,
8690 bool IsSigned) {
8692 I.getType(), true);
8693 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
8694 setValue(&I, Value);
8695}
8696
8697/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
8698/// true and lower it. Otherwise return false, and it will be lowered like a
8699/// normal call.
8700/// The caller already checked that \p I calls the appropriate LibFunc with a
8701/// correct prototype.
8702bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
8703 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
8704 const Value *Size = I.getArgOperand(2);
8705 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
8706 if (CSize && CSize->getZExtValue() == 0) {
8708 I.getType(), true);
8709 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
8710 return true;
8711 }
8712
8714 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
8715 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
8717 if (Res.first.getNode()) {
8718 processIntegerCallValue(I, Res.first, true);
8719 PendingLoads.push_back(Res.second);
8720 return true;
8721 }
8722
8723 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
8724 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
8725 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
8726 return false;
8727
8728 // If the target has a fast compare for the given size, it will return a
8729 // preferred load type for that size. Require that the load VT is legal and
8730 // that the target supports unaligned loads of that type. Otherwise, return
8731 // INVALID.
8732 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
8734 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
8735 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
8736 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
8737 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
8738 // TODO: Check alignment of src and dest ptrs.
8739 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
8740 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
8741 if (!TLI.isTypeLegal(LVT) ||
8742 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
8743 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
8745 }
8746
8747 return LVT;
8748 };
8749
8750 // This turns into unaligned loads. We only do this if the target natively
8751 // supports the MVT we'll be loading or if it is small enough (<= 4) that
8752 // we'll only produce a small number of byte loads.
8753 MVT LoadVT;
8754 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
8755 switch (NumBitsToCompare) {
8756 default:
8757 return false;
8758 case 16:
8759 LoadVT = MVT::i16;
8760 break;
8761 case 32:
8762 LoadVT = MVT::i32;
8763 break;
8764 case 64:
8765 case 128:
8766 case 256:
8767 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
8768 break;
8769 }
8770
8771 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
8772 return false;
8773
8774 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
8775 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
8776
8777 // Bitcast to a wide integer type if the loads are vectors.
8778 if (LoadVT.isVector()) {
8779 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
8780 LoadL = DAG.getBitcast(CmpVT, LoadL);
8781 LoadR = DAG.getBitcast(CmpVT, LoadR);
8782 }
8783
8784 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
8785 processIntegerCallValue(I, Cmp, false);
8786 return true;
8787}
8788
8789/// See if we can lower a memchr call into an optimized form. If so, return
8790/// true and lower it. Otherwise return false, and it will be lowered like a
8791/// normal call.
8792/// The caller already checked that \p I calls the appropriate LibFunc with a
8793/// correct prototype.
8794bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
8795 const Value *Src = I.getArgOperand(0);
8796 const Value *Char = I.getArgOperand(1);
8797 const Value *Length = I.getArgOperand(2);
8798
8800 std::pair<SDValue, SDValue> Res =
8802 getValue(Src), getValue(Char), getValue(Length),
8803 MachinePointerInfo(Src));
8804 if (Res.first.getNode()) {
8805 setValue(&I, Res.first);
8806 PendingLoads.push_back(Res.second);
8807 return true;
8808 }
8809
8810 return false;
8811}
8812
8813/// See if we can lower a mempcpy call into an optimized form. If so, return
8814/// true and lower it. Otherwise return false, and it will be lowered like a
8815/// normal call.
8816/// The caller already checked that \p I calls the appropriate LibFunc with a
8817/// correct prototype.
8818bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
8819 SDValue Dst = getValue(I.getArgOperand(0));
8820 SDValue Src = getValue(I.getArgOperand(1));
8821 SDValue Size = getValue(I.getArgOperand(2));
8822
8823 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
8824 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
8825 // DAG::getMemcpy needs Alignment to be defined.
8826 Align Alignment = std::min(DstAlign, SrcAlign);
8827
8828 SDLoc sdl = getCurSDLoc();
8829
8830 // In the mempcpy context we need to pass in a false value for isTailCall
8831 // because the return pointer needs to be adjusted by the size of
8832 // the copied memory.
8833 SDValue Root = getMemoryRoot();
8834 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, false, false,
8835 /*isTailCall=*/false,
8836 MachinePointerInfo(I.getArgOperand(0)),
8837 MachinePointerInfo(I.getArgOperand(1)),
8838 I.getAAMetadata());
8839 assert(MC.getNode() != nullptr &&
8840 "** memcpy should not be lowered as TailCall in mempcpy context **");
8841 DAG.setRoot(MC);
8842
8843 // Check if Size needs to be truncated or extended.
8844 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
8845
8846 // Adjust return pointer to point just past the last dst byte.
8847 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
8848 Dst, Size);
8849 setValue(&I, DstPlusSize);
8850 return true;
8851}
8852
8853/// See if we can lower a strcpy call into an optimized form. If so, return
8854/// true and lower it, otherwise return false and it will be lowered like a
8855/// normal call.
8856/// The caller already checked that \p I calls the appropriate LibFunc with a
8857/// correct prototype.
8858bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
8859 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8860
8862 std::pair<SDValue, SDValue> Res =
8864 getValue(Arg0), getValue(Arg1),
8865 MachinePointerInfo(Arg0),
8866 MachinePointerInfo(Arg1), isStpcpy);
8867 if (Res.first.getNode()) {
8868 setValue(&I, Res.first);
8869 DAG.setRoot(Res.second);
8870 return true;
8871 }
8872
8873 return false;
8874}
8875
8876/// See if we can lower a strcmp call into an optimized form. If so, return
8877/// true and lower it, otherwise return false and it will be lowered like a
8878/// normal call.
8879/// The caller already checked that \p I calls the appropriate LibFunc with a
8880/// correct prototype.
8881bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
8882 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8883
8885 std::pair<SDValue, SDValue> Res =
8887 getValue(Arg0), getValue(Arg1),
8888 MachinePointerInfo(Arg0),
8889 MachinePointerInfo(Arg1));
8890 if (Res.first.getNode()) {
8891 processIntegerCallValue(I, Res.first, true);
8892 PendingLoads.push_back(Res.second);
8893 return true;
8894 }
8895
8896 return false;
8897}
8898
8899/// See if we can lower a strlen call into an optimized form. If so, return
8900/// true and lower it, otherwise return false and it will be lowered like a
8901/// normal call.
8902/// The caller already checked that \p I calls the appropriate LibFunc with a
8903/// correct prototype.
8904bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
8905 const Value *Arg0 = I.getArgOperand(0);
8906
8908 std::pair<SDValue, SDValue> Res =
8910 getValue(Arg0), MachinePointerInfo(Arg0));
8911 if (Res.first.getNode()) {
8912 processIntegerCallValue(I, Res.first, false);
8913 PendingLoads.push_back(Res.second);
8914 return true;
8915 }
8916
8917 return false;
8918}
8919
8920/// See if we can lower a strnlen call into an optimized form. If so, return
8921/// true and lower it, otherwise return false and it will be lowered like a
8922/// normal call.
8923/// The caller already checked that \p I calls the appropriate LibFunc with a
8924/// correct prototype.
8925bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
8926 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8927
8929 std::pair<SDValue, SDValue> Res =
8931 getValue(Arg0), getValue(Arg1),
8932 MachinePointerInfo(Arg0));
8933 if (Res.first.getNode()) {
8934 processIntegerCallValue(I, Res.first, false);
8935 PendingLoads.push_back(Res.second);
8936 return true;
8937 }
8938
8939 return false;
8940}
8941
8942/// See if we can lower a unary floating-point operation into an SDNode with
8943/// the specified Opcode. If so, return true and lower it, otherwise return
8944/// false and it will be lowered like a normal call.
8945/// The caller already checked that \p I calls the appropriate LibFunc with a
8946/// correct prototype.
8947bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
8948 unsigned Opcode) {
8949 // We already checked this call's prototype; verify it doesn't modify errno.
8950 if (!I.onlyReadsMemory())
8951 return false;
8952
8954 Flags.copyFMF(cast<FPMathOperator>(I));
8955
8956 SDValue Tmp = getValue(I.getArgOperand(0));
8957 setValue(&I,
8958 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
8959 return true;
8960}
8961
8962/// See if we can lower a binary floating-point operation into an SDNode with
8963/// the specified Opcode. If so, return true and lower it. Otherwise return
8964/// false, and it will be lowered like a normal call.
8965/// The caller already checked that \p I calls the appropriate LibFunc with a
8966/// correct prototype.
8967bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
8968 unsigned Opcode) {
8969 // We already checked this call's prototype; verify it doesn't modify errno.
8970 if (!I.onlyReadsMemory())
8971 return false;
8972
8974 Flags.copyFMF(cast<FPMathOperator>(I));
8975
8976 SDValue Tmp0 = getValue(I.getArgOperand(0));
8977 SDValue Tmp1 = getValue(I.getArgOperand(1));
8978 EVT VT = Tmp0.getValueType();
8979 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
8980 return true;
8981}
8982
8983void SelectionDAGBuilder::visitCall(const CallInst &I) {
8984 // Handle inline assembly differently.
8985 if (I.isInlineAsm()) {
8986 visitInlineAsm(I);
8987 return;
8988 }
8989
8991
8992 if (Function *F = I.getCalledFunction()) {
8993 if (F->isDeclaration()) {
8994 // Is this an LLVM intrinsic or a target-specific intrinsic?
8995 unsigned IID = F->getIntrinsicID();
8996 if (!IID)
8997 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
8998 IID = II->getIntrinsicID(F);
8999
9000 if (IID) {
9001 visitIntrinsicCall(I, IID);
9002 return;
9003 }
9004 }
9005
9006 // Check for well-known libc/libm calls. If the function is internal, it
9007 // can't be a library call. Don't do the check if marked as nobuiltin for
9008 // some reason or the call site requires strict floating point semantics.
9009 LibFunc Func;
9010 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
9011 F->hasName() && LibInfo->getLibFunc(*F, Func) &&
9013 switch (Func) {
9014 default: break;
9015 case LibFunc_bcmp:
9016 if (visitMemCmpBCmpCall(I))
9017 return;
9018 break;
9019 case LibFunc_copysign:
9020 case LibFunc_copysignf:
9021 case LibFunc_copysignl:
9022 // We already checked this call's prototype; verify it doesn't modify
9023 // errno.
9024 if (I.onlyReadsMemory()) {
9025 SDValue LHS = getValue(I.getArgOperand(0));
9026 SDValue RHS = getValue(I.getArgOperand(1));
9028 LHS.getValueType(), LHS, RHS));
9029 return;
9030 }
9031 break;
9032 case LibFunc_fabs:
9033 case LibFunc_fabsf:
9034 case LibFunc_fabsl:
9035 if (visitUnaryFloatCall(I, ISD::FABS))
9036 return;
9037 break;
9038 case LibFunc_fmin:
9039 case LibFunc_fminf:
9040 case LibFunc_fminl:
9041 if (visitBinaryFloatCall(I, ISD::FMINNUM))
9042 return;
9043 break;
9044 case LibFunc_fmax:
9045 case LibFunc_fmaxf:
9046 case LibFunc_fmaxl:
9047 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
9048 return;
9049 break;
9050 case LibFunc_sin:
9051 case LibFunc_sinf:
9052 case LibFunc_sinl:
9053 if (visitUnaryFloatCall(I, ISD::FSIN))
9054 return;
9055 break;
9056 case LibFunc_cos:
9057 case LibFunc_cosf:
9058 case LibFunc_cosl:
9059 if (visitUnaryFloatCall(I, ISD::FCOS))
9060 return;
9061 break;
9062 case LibFunc_sqrt:
9063 case LibFunc_sqrtf:
9064 case LibFunc_sqrtl:
9065 case LibFunc_sqrt_finite:
9066 case LibFunc_sqrtf_finite:
9067 case LibFunc_sqrtl_finite:
9068 if (visitUnaryFloatCall(I, ISD::FSQRT))
9069 return;
9070 break;
9071 case LibFunc_floor:
9072 case LibFunc_floorf:
9073 case LibFunc_floorl:
9074 if (visitUnaryFloatCall(I, ISD::FFLOOR))
9075 return;
9076 break;
9077 case LibFunc_nearbyint:
9078 case LibFunc_nearbyintf:
9079 case LibFunc_nearbyintl:
9080 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
9081 return;
9082 break;
9083 case LibFunc_ceil:
9084 case LibFunc_ceilf:
9085 case LibFunc_ceill:
9086 if (visitUnaryFloatCall(I, ISD::FCEIL))
9087 return;
9088 break;
9089 case LibFunc_rint:
9090 case LibFunc_rintf:
9091 case LibFunc_rintl:
9092 if (visitUnaryFloatCall(I, ISD::FRINT))
9093 return;
9094 break;
9095 case LibFunc_round:
9096 case LibFunc_roundf:
9097 case LibFunc_roundl:
9098 if (visitUnaryFloatCall(I, ISD::FROUND))
9099 return;
9100 break;
9101 case LibFunc_trunc:
9102 case LibFunc_truncf:
9103 case LibFunc_truncl:
9104 if (visitUnaryFloatCall(I, ISD::FTRUNC))
9105 return;
9106 break;
9107 case LibFunc_log2:
9108 case LibFunc_log2f:
9109 case LibFunc_log2l:
9110 if (visitUnaryFloatCall(I, ISD::FLOG2))
9111 return;
9112 break;
9113 case LibFunc_exp2:
9114 case LibFunc_exp2f:
9115 case LibFunc_exp2l:
9116 if (visitUnaryFloatCall(I, ISD::FEXP2))
9117 return;
9118 break;
9119 case LibFunc_exp10:
9120 case LibFunc_exp10f:
9121 case LibFunc_exp10l:
9122 if (visitUnaryFloatCall(I, ISD::FEXP10))
9123 return;
9124 break;
9125 case LibFunc_ldexp:
9126 case LibFunc_ldexpf:
9127 case LibFunc_ldexpl:
9128 if (visitBinaryFloatCall(I, ISD::FLDEXP))
9129 return;
9130 break;
9131 case LibFunc_memcmp:
9132 if (visitMemCmpBCmpCall(I))
9133 return;
9134 break;
9135 case LibFunc_mempcpy:
9136 if (visitMemPCpyCall(I))
9137 return;
9138 break;
9139 case LibFunc_memchr:
9140 if (visitMemChrCall(I))
9141 return;
9142 break;
9143 case LibFunc_strcpy:
9144 if (visitStrCpyCall(I, false))
9145 return;
9146 break;
9147 case LibFunc_stpcpy:
9148 if (visitStrCpyCall(I, true))
9149 return;
9150 break;
9151 case LibFunc_strcmp:
9152 if (visitStrCmpCall(I))
9153 return;
9154 break;
9155 case LibFunc_strlen:
9156 if (visitStrLenCall(I))
9157 return;
9158 break;
9159 case LibFunc_strnlen:
9160 if (visitStrNLenCall(I))
9161 return;
9162 break;
9163 }
9164 }
9165 }
9166
9167 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9168 // have to do anything here to lower funclet bundles.
9169 // CFGuardTarget bundles are lowered in LowerCallTo.
9170 assert(!I.hasOperandBundlesOtherThan(
9171 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9172 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9173 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9174 LLVMContext::OB_convergencectrl}) &&
9175 "Cannot lower calls with arbitrary operand bundles!");
9176
9177 SDValue Callee = getValue(I.getCalledOperand());
9178
9179 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
9180 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
9181 else
9182 // Check if we can potentially perform a tail call. More detailed checking
9183 // is be done within LowerCallTo, after more information about the call is
9184 // known.
9185 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
9186}
9187
9188namespace {
9189
9190/// AsmOperandInfo - This contains information for each constraint that we are
9191/// lowering.
9192class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9193public:
9194 /// CallOperand - If this is the result output operand or a clobber
9195 /// this is null, otherwise it is the incoming operand to the CallInst.
9196 /// This gets modified as the asm is processed.
9197 SDValue CallOperand;
9198
9199 /// AssignedRegs - If this is a register or register class operand, this
9200 /// contains the set of register corresponding to the operand.
9201 RegsForValue AssignedRegs;
9202
9203 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9204 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9205 }
9206
9207 /// Whether or not this operand accesses memory
9208 bool hasMemory(const TargetLowering &TLI) const {
9209 // Indirect operand accesses access memory.
9210 if (isIndirect)
9211 return true;
9212
9213 for (const auto &Code : Codes)
9215 return true;
9216
9217 return false;
9218 }
9219};
9220
9221
9222} // end anonymous namespace
9223
9224/// Make sure that the output operand \p OpInfo and its corresponding input
9225/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9226/// out).
9227static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9228 SDISelAsmOperandInfo &MatchingOpInfo,
9229 SelectionDAG &DAG) {
9230 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9231 return;
9232
9234 const auto &TLI = DAG.getTargetLoweringInfo();
9235
9236 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9237 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
9238 OpInfo.ConstraintVT);
9239 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9240 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
9241 MatchingOpInfo.ConstraintVT);
9242 if ((OpInfo.ConstraintVT.isInteger() !=
9243 MatchingOpInfo.ConstraintVT.isInteger()) ||
9244 (MatchRC.second != InputRC.second)) {
9245 // FIXME: error out in a more elegant fashion
9246 report_fatal_error("Unsupported asm: input constraint"
9247 " with a matching output constraint of"
9248 " incompatible type!");
9249 }
9250 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9251}
9252
9253/// Get a direct memory input to behave well as an indirect operand.
9254/// This may introduce stores, hence the need for a \p Chain.
9255/// \return The (possibly updated) chain.
9256static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9257 SDISelAsmOperandInfo &OpInfo,
9258 SelectionDAG &DAG) {
9259 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9260
9261 // If we don't have an indirect input, put it in the constpool if we can,
9262 // otherwise spill it to a stack slot.
9263 // TODO: This isn't quite right. We need to handle these according to
9264 // the addressing mode that the constraint wants. Also, this may take
9265 // an additional register for the computation and we don't want that
9266 // either.
9267
9268 // If the operand is a float, integer, or vector constant, spill to a
9269 // constant pool entry to get its address.
9270 const Value *OpVal = OpInfo.CallOperandVal;
9271 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
9272 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
9273 OpInfo.CallOperand = DAG.getConstantPool(
9274 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
9275 return Chain;
9276 }
9277
9278 // Otherwise, create a stack slot and emit a store to it before the asm.
9279 Type *Ty = OpVal->getType();
9280 auto &DL = DAG.getDataLayout();
9281 uint64_t TySize = DL.getTypeAllocSize(Ty);
9283 int SSFI = MF.getFrameInfo().CreateStackObject(
9284 TySize, DL.getPrefTypeAlign(Ty), false);
9285 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
9286 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
9288 TLI.getMemValueType(DL, Ty));
9289 OpInfo.CallOperand = StackSlot;
9290
9291 return Chain;
9292}
9293
9294/// GetRegistersForValue - Assign registers (virtual or physical) for the
9295/// specified operand. We prefer to assign virtual registers, to allow the
9296/// register allocator to handle the assignment process. However, if the asm
9297/// uses features that we can't model on machineinstrs, we have SDISel do the
9298/// allocation. This produces generally horrible, but correct, code.
9299///
9300/// OpInfo describes the operand
9301/// RefOpInfo describes the matching operand if any, the operand otherwise
9302static std::optional<unsigned>
9304 SDISelAsmOperandInfo &OpInfo,
9305 SDISelAsmOperandInfo &RefOpInfo) {
9306 LLVMContext &Context = *DAG.getContext();
9307 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9308
9312
9313 // No work to do for memory/address operands.
9314 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9315 OpInfo.ConstraintType == TargetLowering::C_Address)
9316 return std::nullopt;
9317
9318 // If this is a constraint for a single physreg, or a constraint for a
9319 // register class, find it.
9320 unsigned AssignedReg;
9321 const TargetRegisterClass *RC;
9322 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
9323 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
9324 // RC is unset only on failure. Return immediately.
9325 if (!RC)
9326 return std::nullopt;
9327
9328 // Get the actual register value type. This is important, because the user
9329 // may have asked for (e.g.) the AX register in i32 type. We need to
9330 // remember that AX is actually i16 to get the right extension.
9331 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
9332
9333 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9334 // If this is an FP operand in an integer register (or visa versa), or more
9335 // generally if the operand value disagrees with the register class we plan
9336 // to stick it in, fix the operand type.
9337 //
9338 // If this is an input value, the bitcast to the new type is done now.
9339 // Bitcast for output value is done at the end of visitInlineAsm().
9340 if ((OpInfo.Type == InlineAsm::isOutput ||
9341 OpInfo.Type == InlineAsm::isInput) &&
9342 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
9343 // Try to convert to the first EVT that the reg class contains. If the
9344 // types are identical size, use a bitcast to convert (e.g. two differing
9345 // vector types). Note: output bitcast is done at the end of
9346 // visitInlineAsm().
9347 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9348 // Exclude indirect inputs while they are unsupported because the code
9349 // to perform the load is missing and thus OpInfo.CallOperand still
9350 // refers to the input address rather than the pointed-to value.
9351 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
9352 OpInfo.CallOperand =
9353 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
9354 OpInfo.ConstraintVT = RegVT;
9355 // If the operand is an FP value and we want it in integer registers,
9356 // use the corresponding integer type. This turns an f64 value into
9357 // i64, which can be passed with two i32 values on a 32-bit machine.
9358 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9359 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
9360 if (OpInfo.Type == InlineAsm::isInput)
9361 OpInfo.CallOperand =
9362 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
9363 OpInfo.ConstraintVT = VT;
9364 }
9365 }
9366 }
9367
9368 // No need to allocate a matching input constraint since the constraint it's
9369 // matching to has already been allocated.
9370 if (OpInfo.isMatchingInputConstraint())
9371 return std::nullopt;
9372
9373 EVT ValueVT = OpInfo.ConstraintVT;
9374 if (OpInfo.ConstraintVT == MVT::Other)
9375 ValueVT = RegVT;
9376
9377 // Initialize NumRegs.
9378 unsigned NumRegs = 1;
9379 if (OpInfo.ConstraintVT != MVT::Other)
9380 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
9381
9382 // If this is a constraint for a specific physical register, like {r17},
9383 // assign it now.
9384
9385 // If this associated to a specific register, initialize iterator to correct
9386 // place. If virtual, make sure we have enough registers
9387
9388 // Initialize iterator if necessary
9391
9392 // Do not check for single registers.
9393 if (AssignedReg) {
9394 I = std::find(I, RC->end(), AssignedReg);
9395 if (I == RC->end()) {
9396 // RC does not contain the selected register, which indicates a
9397 // mismatch between the register and the required type/bitwidth.
9398 return {AssignedReg};
9399 }
9400 }
9401
9402 for (; NumRegs; --NumRegs, ++I) {
9403 assert(I != RC->end() && "Ran out of registers to allocate!");
9404 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
9405 Regs.push_back(R);
9406 }
9407
9408 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
9409 return std::nullopt;
9410}
9411
9412static unsigned
9414 const std::vector<SDValue> &AsmNodeOperands) {
9415 // Scan until we find the definition we already emitted of this operand.
9416 unsigned CurOp = InlineAsm::Op_FirstOperand;
9417 for (; OperandNo; --OperandNo) {
9418 // Advance to the next operand.
9419 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9420 const InlineAsm::Flag F(OpFlag);
9421 assert(
9422 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
9423 "Skipped past definitions?");
9424 CurOp += F.getNumOperandRegisters() + 1;
9425 }
9426 return CurOp;
9427}
9428
9429namespace {
9430
9431class ExtraFlags {
9432 unsigned Flags = 0;
9433
9434public:
9435 explicit ExtraFlags(const CallBase &Call) {
9436 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9437 if (IA->hasSideEffects())
9439 if (IA->isAlignStack())
9441 if (Call.isConvergent())
9443 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
9444 }
9445
9446 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
9447 // Ideally, we would only check against memory constraints. However, the
9448 // meaning of an Other constraint can be target-specific and we can't easily
9449 // reason about it. Therefore, be conservative and set MayLoad/MayStore
9450 // for Other constraints as well.
9453 if (OpInfo.Type == InlineAsm::isInput)
9455 else if (OpInfo.Type == InlineAsm::isOutput)
9457 else if (OpInfo.Type == InlineAsm::isClobber)
9459 }
9460 }
9461
9462 unsigned get() const { return Flags; }
9463};
9464
9465} // end anonymous namespace
9466
9467static bool isFunction(SDValue Op) {
9468 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
9469 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
9470 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
9471
9472 // In normal "call dllimport func" instruction (non-inlineasm) it force
9473 // indirect access by specifing call opcode. And usually specially print
9474 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
9475 // not do in this way now. (In fact, this is similar with "Data Access"
9476 // action). So here we ignore dllimport function.
9477 if (Fn && !Fn->hasDLLImportStorageClass())
9478 return true;
9479 }
9480 }
9481 return false;
9482}
9483
9484/// visitInlineAsm - Handle a call to an InlineAsm object.
9485void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
9486 const BasicBlock *EHPadBB) {
9487 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9488
9489 /// ConstraintOperands - Information about all of the constraints.
9490 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
9491
9495
9496 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
9497 // AsmDialect, MayLoad, MayStore).
9498 bool HasSideEffect = IA->hasSideEffects();
9499 ExtraFlags ExtraInfo(Call);
9500
9501 for (auto &T : TargetConstraints) {
9502 ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
9503 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
9504
9505 if (OpInfo.CallOperandVal)
9506 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
9507
9508 if (!HasSideEffect)
9509 HasSideEffect = OpInfo.hasMemory(TLI);
9510
9511 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
9512 // FIXME: Could we compute this on OpInfo rather than T?
9513
9514 // Compute the constraint code and ConstraintType to use.
9516
9517 if (T.ConstraintType == TargetLowering::C_Immediate &&
9518 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
9519 // We've delayed emitting a diagnostic like the "n" constraint because
9520 // inlining could cause an integer showing up.
9521 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
9522 "' expects an integer constant "
9523 "expression");
9524
9525 ExtraInfo.update(T);
9526 }
9527
9528 // We won't need to flush pending loads if this asm doesn't touch
9529 // memory and is nonvolatile.
9530 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
9531
9532 bool EmitEHLabels = isa<InvokeInst>(Call);
9533 if (EmitEHLabels) {
9534 assert(EHPadBB && "InvokeInst must have an EHPadBB");
9535 }
9536 bool IsCallBr = isa<CallBrInst>(Call);
9537
9538 if (IsCallBr || EmitEHLabels) {
9539 // If this is a callbr or invoke we need to flush pending exports since
9540 // inlineasm_br and invoke are terminators.
9541 // We need to do this before nodes are glued to the inlineasm_br node.
9542 Chain = getControlRoot();
9543 }
9544
9545 MCSymbol *BeginLabel = nullptr;
9546 if (EmitEHLabels) {
9547 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
9548 }
9549
9550 int OpNo = -1;
9551 SmallVector<StringRef> AsmStrs;
9552 IA->collectAsmStrs(AsmStrs);
9553
9554 // Second pass over the constraints: compute which constraint option to use.
9555 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9556 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
9557 OpNo++;
9558
9559 // If this is an output operand with a matching input operand, look up the
9560 // matching input. If their types mismatch, e.g. one is an integer, the
9561 // other is floating point, or their sizes are different, flag it as an
9562 // error.
9563 if (OpInfo.hasMatchingInput()) {
9564 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
9565 patchMatchingInput(OpInfo, Input, DAG);
9566 }
9567
9568 // Compute the constraint code and ConstraintType to use.
9569 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
9570
9571 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
9572 OpInfo.Type == InlineAsm::isClobber) ||
9573 OpInfo.ConstraintType == TargetLowering::C_Address)
9574 continue;
9575
9576 // In Linux PIC model, there are 4 cases about value/label addressing:
9577 //
9578 // 1: Function call or Label jmp inside the module.
9579 // 2: Data access (such as global variable, static variable) inside module.
9580 // 3: Function call or Label jmp outside the module.
9581 // 4: Data access (such as global variable) outside the module.
9582 //
9583 // Due to current llvm inline asm architecture designed to not "recognize"
9584 // the asm code, there are quite troubles for us to treat mem addressing
9585 // differently for same value/adress used in different instuctions.
9586 // For example, in pic model, call a func may in plt way or direclty
9587 // pc-related, but lea/mov a function adress may use got.
9588 //
9589 // Here we try to "recognize" function call for the case 1 and case 3 in
9590 // inline asm. And try to adjust the constraint for them.
9591 //
9592 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
9593 // label, so here we don't handle jmp function label now, but we need to
9594 // enhance it (especilly in PIC model) if we meet meaningful requirements.
9595 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
9596 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
9598 OpInfo.isIndirect = false;
9599 OpInfo.ConstraintType = TargetLowering::C_Address;
9600 }
9601
9602 // If this is a memory input, and if the operand is not indirect, do what we
9603 // need to provide an address for the memory input.
9604 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
9605 !OpInfo.isIndirect) {
9606 assert((OpInfo.isMultipleAlternative ||
9607 (OpInfo.Type == InlineAsm::isInput)) &&
9608 "Can only indirectify direct input operands!");
9609
9610 // Memory operands really want the address of the value.
9611 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
9612
9613 // There is no longer a Value* corresponding to this operand.
9614 OpInfo.CallOperandVal = nullptr;
9615
9616 // It is now an indirect operand.
9617 OpInfo.isIndirect = true;
9618 }
9619
9620 }
9621
9622 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
9623 std::vector<SDValue> AsmNodeOperands;
9624 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
9625 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
9626 IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
9627
9628 // If we have a !srcloc metadata node associated with it, we want to attach
9629 // this to the ultimately generated inline asm machineinstr. To do this, we
9630 // pass in the third operand as this (potentially null) inline asm MDNode.
9631 const MDNode *SrcLoc = Call.getMetadata("srcloc");
9632 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
9633
9634 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
9635 // bits as operand 3.
9636 AsmNodeOperands.push_back(DAG.getTargetConstant(
9637 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9638
9639 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
9640 // this, assign virtual and physical registers for inputs and otput.
9641 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9642 // Assign Registers.
9643 SDISelAsmOperandInfo &RefOpInfo =
9644 OpInfo.isMatchingInputConstraint()
9645 ? ConstraintOperands[OpInfo.getMatchedOperand()]
9646 : OpInfo;
9647 const auto RegError =
9648 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
9649 if (RegError) {
9652 const char *RegName = TRI.getName(*RegError);
9653 emitInlineAsmError(Call, "register '" + Twine(RegName) +
9654 "' allocated for constraint '" +
9655 Twine(OpInfo.ConstraintCode) +
9656 "' does not match required type");
9657 return;
9658 }
9659
9660 auto DetectWriteToReservedRegister = [&]() {
9663 for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
9665 TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
9666 const char *RegName = TRI.getName(Reg);
9667 emitInlineAsmError(Call, "write to reserved register '" +
9668 Twine(RegName) + "'");
9669 return true;
9670 }
9671 }
9672 return false;
9673 };
9674 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
9675 (OpInfo.Type == InlineAsm::isInput &&
9676 !OpInfo.isMatchingInputConstraint())) &&
9677 "Only address as input operand is allowed.");
9678
9679 switch (OpInfo.Type) {
9681 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9682 const InlineAsm::ConstraintCode ConstraintID =
9683 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9685 "Failed to convert memory constraint code to constraint id.");
9686
9687 // Add information to the INLINEASM node to know about this output.
9689 OpFlags.setMemConstraint(ConstraintID);
9690 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
9691 MVT::i32));
9692 AsmNodeOperands.push_back(OpInfo.CallOperand);
9693 } else {
9694 // Otherwise, this outputs to a register (directly for C_Register /
9695 // C_RegisterClass, and a target-defined fashion for
9696 // C_Immediate/C_Other). Find a register that we can use.
9697 if (OpInfo.AssignedRegs.Regs.empty()) {
9698 emitInlineAsmError(
9699 Call, "couldn't allocate output register for constraint '" +
9700 Twine(OpInfo.ConstraintCode) + "'");
9701 return;
9702 }
9703
9704 if (DetectWriteToReservedRegister())
9705 return;
9706
9707 // Add information to the INLINEASM node to know that this register is
9708 // set.
9709 OpInfo.AssignedRegs.AddInlineAsmOperands(
9710 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
9712 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
9713 }
9714 break;
9715
9716 case InlineAsm::isInput:
9717 case InlineAsm::isLabel: {
9718 SDValue InOperandVal = OpInfo.CallOperand;
9719
9720 if (OpInfo.isMatchingInputConstraint()) {
9721 // If this is required to match an output register we have already set,
9722 // just use its register.
9723 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
9724 AsmNodeOperands);
9725 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
9726 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
9727 if (OpInfo.isIndirect) {
9728 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
9729 emitInlineAsmError(Call, "inline asm not supported yet: "
9730 "don't know how to handle tied "
9731 "indirect register inputs");
9732 return;
9733 }
9734
9739 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
9740 Register TiedReg = R->getReg();
9741 MVT RegVT = R->getSimpleValueType(0);
9742 const TargetRegisterClass *RC =
9743 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)
9744 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
9745 : TRI.getMinimalPhysRegClass(TiedReg);
9746 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
9747 Regs.push_back(MRI.createVirtualRegister(RC));
9748
9749 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
9750
9751 SDLoc dl = getCurSDLoc();
9752 // Use the produced MatchedRegs object to
9753 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);
9754 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, true,
9755 OpInfo.getMatchedOperand(), dl, DAG,
9756 AsmNodeOperands);
9757 break;
9758 }
9759
9760 assert(Flag.isMemKind() && "Unknown matching constraint!");
9761 assert(Flag.getNumOperandRegisters() == 1 &&
9762 "Unexpected number of operands");
9763 // Add information to the INLINEASM node to know about this input.
9764 // See InlineAsm.h isUseOperandTiedToDef.
9765 Flag.clearMemConstraint();
9766 Flag.setMatchingOp(OpInfo.getMatchedOperand());
9767 AsmNodeOperands.push_back(DAG.getTargetConstant(
9768 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9769 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
9770 break;
9771 }
9772
9773 // Treat indirect 'X' constraint as memory.
9774 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
9775 OpInfo.isIndirect)
9776 OpInfo.ConstraintType = TargetLowering::C_Memory;
9777
9778 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
9779 OpInfo.ConstraintType == TargetLowering::C_Other) {
9780 std::vector<SDValue> Ops;
9781 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
9782 Ops, DAG);
9783 if (Ops.empty()) {
9784 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
9785 if (isa<ConstantSDNode>(InOperandVal)) {
9786 emitInlineAsmError(Call, "value out of range for constraint '" +
9787 Twine(OpInfo.ConstraintCode) + "'");
9788 return;
9789 }
9790
9791 emitInlineAsmError(Call,
9792 "invalid operand for inline asm constraint '" +
9793 Twine(OpInfo.ConstraintCode) + "'");
9794 return;
9795 }
9796
9797 // Add information to the INLINEASM node to know about this input.
9798 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
9799 AsmNodeOperands.push_back(DAG.getTargetConstant(
9800 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9801 llvm::append_range(AsmNodeOperands, Ops);
9802 break;
9803 }
9804
9805 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9806 assert((OpInfo.isIndirect ||
9807 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
9808 "Operand must be indirect to be a mem!");
9809 assert(InOperandVal.getValueType() ==
9811 "Memory operands expect pointer values");
9812
9813 const InlineAsm::ConstraintCode ConstraintID =
9814 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9816 "Failed to convert memory constraint code to constraint id.");
9817
9818 // Add information to the INLINEASM node to know about this input.
9820 ResOpType.setMemConstraint(ConstraintID);
9821 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
9822 getCurSDLoc(),
9823 MVT::i32));
9824 AsmNodeOperands.push_back(InOperandVal);
9825 break;
9826 }
9827
9828 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
9829 const InlineAsm::ConstraintCode ConstraintID =
9830 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9832 "Failed to convert memory constraint code to constraint id.");
9833
9835
9836 SDValue AsmOp = InOperandVal;
9837 if (isFunction(InOperandVal)) {
9838 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
9839 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
9840 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
9841 InOperandVal.getValueType(),
9842 GA->getOffset());
9843 }
9844
9845 // Add information to the INLINEASM node to know about this input.
9846 ResOpType.setMemConstraint(ConstraintID);
9847
9848 AsmNodeOperands.push_back(
9849 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
9850
9851 AsmNodeOperands.push_back(AsmOp);
9852 break;
9853 }
9854
9855 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
9856 OpInfo.ConstraintType == TargetLowering::C_Register) &&
9857 "Unknown constraint type!");
9858
9859 // TODO: Support this.
9860 if (OpInfo.isIndirect) {
9861 emitInlineAsmError(
9862 Call, "Don't know how to handle indirect register inputs yet "
9863 "for constraint '" +
9864 Twine(OpInfo.ConstraintCode) + "'");
9865 return;
9866 }
9867
9868 // Copy the input into the appropriate registers.
9869 if (OpInfo.AssignedRegs.Regs.empty()) {
9870 emitInlineAsmError(Call,
9871 "couldn't allocate input reg for constraint '" +
9872 Twine(OpInfo.ConstraintCode) + "'");
9873 return;
9874 }
9875
9876 if (DetectWriteToReservedRegister())
9877 return;
9878
9879 SDLoc dl = getCurSDLoc();
9880
9881 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,
9882 &Call);
9883
9884 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, false,
9885 0, dl, DAG, AsmNodeOperands);
9886 break;
9887 }
9889 // Add the clobbered value to the operand list, so that the register
9890 // allocator is aware that the physreg got clobbered.
9891 if (!OpInfo.AssignedRegs.Regs.empty())
9892 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::Clobber,
9893 false, 0, getCurSDLoc(), DAG,
9894 AsmNodeOperands);
9895 break;
9896 }
9897 }
9898
9899 // Finish up input operands. Set the input chain and add the flag last.
9900 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
9901 if (Glue.getNode()) AsmNodeOperands.push_back(Glue);
9902
9903 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
9904 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
9905 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
9906 Glue = Chain.getValue(1);
9907
9908 // Do additional work to generate outputs.
9909
9910 SmallVector<EVT, 1> ResultVTs;
9911 SmallVector<SDValue, 1> ResultValues;
9912 SmallVector<SDValue, 8> OutChains;
9913
9914 llvm::Type *CallResultType = Call.getType();
9915 ArrayRef<Type *> ResultTypes;
9916 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
9917 ResultTypes = StructResult->elements();
9918 else if (!CallResultType->isVoidTy())
9919 ResultTypes = ArrayRef(CallResultType);
9920
9921 auto CurResultType = ResultTypes.begin();
9922 auto handleRegAssign = [&](SDValue V) {
9923 assert(CurResultType != ResultTypes.end() && "Unexpected value");
9924 assert((*CurResultType)->isSized() && "Unexpected unsized type");
9925 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
9926 ++CurResultType;
9927 // If the type of the inline asm call site return value is different but has
9928 // same size as the type of the asm output bitcast it. One example of this
9929 // is for vectors with different width / number of elements. This can
9930 // happen for register classes that can contain multiple different value
9931 // types. The preg or vreg allocated may not have the same VT as was
9932 // expected.
9933 //
9934 // This can also happen for a return value that disagrees with the register
9935 // class it is put in, eg. a double in a general-purpose register on a
9936 // 32-bit machine.
9937 if (ResultVT != V.getValueType() &&
9938 ResultVT.getSizeInBits() == V.getValueSizeInBits())
9939 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
9940 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
9941 V.getValueType().isInteger()) {
9942 // If a result value was tied to an input value, the computed result
9943 // may have a wider width than the expected result. Extract the
9944 // relevant portion.
9945 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
9946 }
9947 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
9948 ResultVTs.push_back(ResultVT);
9949 ResultValues.push_back(V);
9950 };
9951
9952 // Deal with output operands.
9953 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9954 if (OpInfo.Type == InlineAsm::isOutput) {
9955 SDValue Val;
9956 // Skip trivial output operands.
9957 if (OpInfo.AssignedRegs.Regs.empty())
9958 continue;
9959
9960 switch (OpInfo.ConstraintType) {
9963 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
9964 Chain, &Glue, &Call);
9965 break;
9968 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
9969 OpInfo, DAG);
9970 break;
9972 break; // Already handled.
9974 break; // Silence warning.
9976 assert(false && "Unexpected unknown constraint");
9977 }
9978
9979 // Indirect output manifest as stores. Record output chains.
9980 if (OpInfo.isIndirect) {
9981 const Value *Ptr = OpInfo.CallOperandVal;
9982 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
9983 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
9985 OutChains.push_back(Store);
9986 } else {
9987 // generate CopyFromRegs to associated registers.
9988 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
9989 if (Val.getOpcode() == ISD::MERGE_VALUES) {
9990 for (const SDValue &V : Val->op_values())
9991 handleRegAssign(V);
9992 } else
9993 handleRegAssign(Val);
9994 }
9995 }
9996 }
9997
9998 // Set results.
9999 if (!ResultValues.empty()) {
10000 assert(CurResultType == ResultTypes.end() &&
10001 "Mismatch in number of ResultTypes");
10002 assert(ResultValues.size() == ResultTypes.size() &&
10003 "Mismatch in number of output operands in asm result");
10004
10006 DAG.getVTList(ResultVTs), ResultValues);
10007 setValue(&Call, V);
10008 }
10009
10010 // Collect store chains.
10011 if (!OutChains.empty())
10012 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
10013
10014 if (EmitEHLabels) {
10015 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
10016 }
10017
10018 // Only Update Root if inline assembly has a memory effect.
10019 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10020 EmitEHLabels)
10021 DAG.setRoot(Chain);
10022}
10023
10024void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10025 const Twine &Message) {
10026 LLVMContext &Ctx = *DAG.getContext();
10027 Ctx.emitError(&Call, Message);
10028
10029 // Make sure we leave the DAG in a valid state
10031 SmallVector<EVT, 1> ValueVTs;
10032 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
10033
10034 if (ValueVTs.empty())
10035 return;
10036
10038 for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
10039 Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
10040
10041 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
10042}
10043
10044void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10046 MVT::Other, getRoot(),
10047 getValue(I.getArgOperand(0)),
10048 DAG.getSrcValue(I.getArgOperand(0))));
10049}
10050
10051void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10053 const DataLayout &DL = DAG.getDataLayout();
10055 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
10056 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
10057 DL.getABITypeAlign(I.getType()).value());
10058 DAG.setRoot(V.getValue(1));
10059
10060 if (I.getType()->isPointerTy())
10062 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
10063 setValue(&I, V);
10064}
10065
10066void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10068 MVT::Other, getRoot(),
10069 getValue(I.getArgOperand(0)),
10070 DAG.getSrcValue(I.getArgOperand(0))));
10071}
10072
10073void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10075 MVT::Other, getRoot(),
10076 getValue(I.getArgOperand(0)),
10077 getValue(I.getArgOperand(1)),
10078 DAG.getSrcValue(I.getArgOperand(0)),
10079 DAG.getSrcValue(I.getArgOperand(1))));
10080}
10081
10083 const Instruction &I,
10084 SDValue Op) {
10085 const MDNode *Range = getRangeMetadata(I);
10086 if (!Range)
10087 return Op;
10088
10090 if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
10091 return Op;
10092
10093 APInt Lo = CR.getUnsignedMin();
10094 if (!Lo.isMinValue())
10095 return Op;
10096
10097 APInt Hi = CR.getUnsignedMax();
10098 unsigned Bits = std::max(Hi.getActiveBits(),
10099 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10100
10101 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
10102
10103 SDLoc SL = getCurSDLoc();
10104
10105 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
10106 DAG.getValueType(SmallVT));
10107 unsigned NumVals = Op.getNode()->getNumValues();
10108 if (NumVals == 1)
10109 return ZExt;
10110
10112
10113 Ops.push_back(ZExt);
10114 for (unsigned I = 1; I != NumVals; ++I)
10115 Ops.push_back(Op.getValue(I));
10116
10117 return DAG.getMergeValues(Ops, SL);
10118}
10119
10120/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10121/// the call being lowered.
10122///
10123/// This is a helper for lowering intrinsics that follow a target calling
10124/// convention or require stack pointer adjustment. Only a subset of the
10125/// intrinsic's operands need to participate in the calling convention.
10128 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10129 AttributeSet RetAttrs, bool IsPatchPoint) {
10131 Args.reserve(NumArgs);
10132
10133 // Populate the argument list.
10134 // Attributes for args start at offset 1, after the return attribute.
10135 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10136 ArgI != ArgE; ++ArgI) {
10137 const Value *V = Call->getOperand(ArgI);
10138
10139 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10140
10142 Entry.Node = getValue(V);
10143 Entry.Ty = V->getType();
10144 Entry.setAttributes(Call, ArgI);
10145 Args.push_back(Entry);
10146 }
10147
10149 .setChain(getRoot())
10150 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
10151 RetAttrs)
10152 .setDiscardResult(Call->use_empty())
10153 .setIsPatchPoint(IsPatchPoint)
10155 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
10156}
10157
10158/// Add a stack map intrinsic call's live variable operands to a stackmap
10159/// or patchpoint target node's operand list.
10160///
10161/// Constants are converted to TargetConstants purely as an optimization to
10162/// avoid constant materialization and register allocation.
10163///
10164/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10165/// generate addess computation nodes, and so FinalizeISel can convert the
10166/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10167/// address materialization and register allocation, but may also be required
10168/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10169/// alloca in the entry block, then the runtime may assume that the alloca's
10170/// StackMap location can be read immediately after compilation and that the
10171/// location is valid at any point during execution (this is similar to the
10172/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10173/// only available in a register, then the runtime would need to trap when
10174/// execution reaches the StackMap in order to read the alloca's location.
10175static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10176 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
10177 SelectionDAGBuilder &Builder) {
10178 SelectionDAG &DAG = Builder.DAG;
10179 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10180 SDValue Op = Builder.getValue(Call.getArgOperand(I));
10181
10182 // Things on the stack are pointer-typed, meaning that they are already
10183 // legal and can be emitted directly to target nodes.
10184 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
10185 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
10186 } else {
10187 // Otherwise emit a target independent node to be legalised.
10188 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
10189 }
10190 }
10191}
10192
10193/// Lower llvm.experimental.stackmap.
10194void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10195 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10196 // [live variables...])
10197
10198 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10199
10200 SDValue Chain, InGlue, Callee;
10202
10203 SDLoc DL = getCurSDLoc();
10205
10206 // The stackmap intrinsic only records the live variables (the arguments
10207 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10208 // intrinsic, this won't be lowered to a function call. This means we don't
10209 // have to worry about calling conventions and target specific lowering code.
10210 // Instead we perform the call lowering right here.
10211 //
10212 // chain, flag = CALLSEQ_START(chain, 0, 0)
10213 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10214 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10215 //
10216 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
10217 InGlue = Chain.getValue(1);
10218
10219 // Add the STACKMAP operands, starting with DAG house-keeping.
10220 Ops.push_back(Chain);
10221 Ops.push_back(InGlue);
10222
10223 // Add the <id>, <numShadowBytes> operands.
10224 //
10225 // These do not require legalisation, and can be emitted directly to target
10226 // constant nodes.
10228 assert(ID.getValueType() == MVT::i64);
10229 SDValue IDConst =
10230 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
10231 Ops.push_back(IDConst);
10232
10233 SDValue Shad = getValue(CI.getArgOperand(1));
10234 assert(Shad.getValueType() == MVT::i32);
10235 SDValue ShadConst =
10237 Ops.push_back(ShadConst);
10238
10239 // Add the live variables.
10240 addStackMapLiveVars(CI, 2, DL, Ops, *this);
10241
10242 // Create the STACKMAP node.
10243 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10244 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
10245 InGlue = Chain.getValue(1);
10246
10247 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
10248
10249 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10250
10251 // Set the root to the target-lowered call chain.
10252 DAG.setRoot(Chain);
10253
10254 // Inform the Frame Information that we have a stackmap in this function.
10256}
10257
10258/// Lower llvm.experimental.patchpoint directly to its target opcode.
10259void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10260 const BasicBlock *EHPadBB) {
10261 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
10262 // i32 <numBytes>,
10263 // i8* <target>,
10264 // i32 <numArgs>,
10265 // [Args...],
10266 // [live variables...])
10267
10269 bool IsAnyRegCC = CC == CallingConv::AnyReg;
10270 bool HasDef = !CB.getType()->isVoidTy();
10271 SDLoc dl = getCurSDLoc();
10273
10274 // Handle immediate and symbolic callees.
10275 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
10276 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
10277 /*isTarget=*/true);
10278 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
10279 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
10280 SDLoc(SymbolicCallee),
10281 SymbolicCallee->getValueType(0));
10282
10283 // Get the real number of arguments participating in the call <numArgs>
10285 unsigned NumArgs = NArgVal->getAsZExtVal();
10286
10287 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
10288 // Intrinsics include all meta-operands up to but not including CC.
10289 unsigned NumMetaOpers = PatchPointOpers::CCPos;
10290 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
10291 "Not enough arguments provided to the patchpoint intrinsic");
10292
10293 // For AnyRegCC the arguments are lowered later on manually.
10294 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10295 Type *ReturnTy =
10296 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
10297
10299 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
10300 ReturnTy, CB.getAttributes().getRetAttrs(), true);
10301 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
10302
10303 SDNode *CallEnd = Result.second.getNode();
10304 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
10305 CallEnd = CallEnd->getOperand(0).getNode();
10306
10307 /// Get a call instruction from the call sequence chain.
10308 /// Tail calls are not allowed.
10309 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
10310 "Expected a callseq node.");
10311 SDNode *Call = CallEnd->getOperand(0).getNode();
10312 bool HasGlue = Call->getGluedNode();
10313
10314 // Replace the target specific call node with the patchable intrinsic.
10316
10317 // Push the chain.
10318 Ops.push_back(*(Call->op_begin()));
10319
10320 // Optionally, push the glue (if any).
10321 if (HasGlue)
10322 Ops.push_back(*(Call->op_end() - 1));
10323
10324 // Push the register mask info.
10325 if (HasGlue)
10326 Ops.push_back(*(Call->op_end() - 2));
10327 else
10328 Ops.push_back(*(Call->op_end() - 1));
10329
10330 // Add the <id> and <numBytes> constants.
10332 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
10334 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
10335
10336 // Add the callee.
10337 Ops.push_back(Callee);
10338
10339 // Adjust <numArgs> to account for any arguments that have been passed on the
10340 // stack instead.
10341 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
10342 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
10343 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10344 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
10345
10346 // Add the calling convention
10347 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
10348
10349 // Add the arguments we omitted previously. The register allocator should
10350 // place these in any free register.
10351 if (IsAnyRegCC)
10352 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
10353 Ops.push_back(getValue(CB.getArgOperand(i)));
10354
10355 // Push the arguments from the call instruction.
10356 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
10357 Ops.append(Call->op_begin() + 2, e);
10358
10359 // Push live variables for the stack map.
10360 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
10361
10362 SDVTList NodeTys;
10363 if (IsAnyRegCC && HasDef) {
10364 // Create the return types based on the intrinsic definition
10366 SmallVector<EVT, 3> ValueVTs;
10367 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
10368 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
10369
10370 // There is always a chain and a glue type at the end
10371 ValueVTs.push_back(MVT::Other);
10372 ValueVTs.push_back(MVT::Glue);
10373 NodeTys = DAG.getVTList(ValueVTs);
10374 } else
10375 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10376
10377 // Replace the target specific call node with a PATCHPOINT node.
10378 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
10379
10380 // Update the NodeMap.
10381 if (HasDef) {
10382 if (IsAnyRegCC)
10383 setValue(&CB, SDValue(PPV.getNode(), 0));
10384 else
10385 setValue(&CB, Result.first);
10386 }
10387
10388 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
10389 // call sequence. Furthermore the location of the chain and glue can change
10390 // when the AnyReg calling convention is used and the intrinsic returns a
10391 // value.
10392 if (IsAnyRegCC && HasDef) {
10393 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
10394 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
10396 } else
10397 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
10398 DAG.DeleteNode(Call);
10399
10400 // Inform the Frame Information that we have a patchpoint in this function.
10402}
10403
10404void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
10405 unsigned Intrinsic) {
10407 SDValue Op1 = getValue(I.getArgOperand(0));
10408 SDValue Op2;
10409 if (I.arg_size() > 1)
10410 Op2 = getValue(I.getArgOperand(1));
10411 SDLoc dl = getCurSDLoc();
10412 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
10413 SDValue Res;
10414 SDNodeFlags SDFlags;
10415 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
10416 SDFlags.copyFMF(*FPMO);
10417
10418 switch (Intrinsic) {
10419 case Intrinsic::vector_reduce_fadd:
10420 if (SDFlags.hasAllowReassociation())
10421 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
10422 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
10423 SDFlags);
10424 else
10425 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
10426 break;
10427 case Intrinsic::vector_reduce_fmul:
10428 if (SDFlags.hasAllowReassociation())
10429 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
10430 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
10431 SDFlags);
10432 else
10433 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
10434 break;
10435 case Intrinsic::vector_reduce_add:
10436 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
10437 break;
10438 case Intrinsic::vector_reduce_mul:
10439 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
10440 break;
10441 case Intrinsic::vector_reduce_and:
10442 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
10443 break;
10444 case Intrinsic::vector_reduce_or:
10445 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
10446 break;
10447 case Intrinsic::vector_reduce_xor:
10448 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
10449 break;
10450 case Intrinsic::vector_reduce_smax:
10451 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
10452 break;
10453 case Intrinsic::vector_reduce_smin:
10454 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
10455 break;
10456 case Intrinsic::vector_reduce_umax:
10457 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
10458 break;
10459 case Intrinsic::vector_reduce_umin:
10460 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
10461 break;
10462 case Intrinsic::vector_reduce_fmax:
10463 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
10464 break;
10465 case Intrinsic::vector_reduce_fmin:
10466 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
10467 break;
10468 case Intrinsic::vector_reduce_fmaximum:
10469 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
10470 break;
10471 case Intrinsic::vector_reduce_fminimum:
10472 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
10473 break;
10474 default:
10475 llvm_unreachable("Unhandled vector reduce intrinsic");
10476 }
10477 setValue(&I, Res);
10478}
10479
10480/// Returns an AttributeList representing the attributes applied to the return
10481/// value of the given call.
10484 if (CLI.RetSExt)
10485 Attrs.push_back(Attribute::SExt);
10486 if (CLI.RetZExt)
10487 Attrs.push_back(Attribute::ZExt);
10488 if (CLI.IsInReg)
10489 Attrs.push_back(Attribute::InReg);
10490
10492 Attrs);
10493}
10494
10495/// TargetLowering::LowerCallTo - This is the default LowerCallTo
10496/// implementation, which just calls LowerCall.
10497/// FIXME: When all targets are
10498/// migrated to using LowerCall, this hook should be integrated into SDISel.
10499std::pair<SDValue, SDValue>
10501 // Handle the incoming return values from the call.
10502 CLI.Ins.clear();
10503 Type *OrigRetTy = CLI.RetTy;
10504 SmallVector<EVT, 4> RetTys;
10506 auto &DL = CLI.DAG.getDataLayout();
10507 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets, 0);
10508
10509 if (CLI.IsPostTypeLegalization) {
10510 // If we are lowering a libcall after legalization, split the return type.
10511 SmallVector<EVT, 4> OldRetTys;
10512 SmallVector<uint64_t, 4> OldOffsets;
10513 RetTys.swap(OldRetTys);
10514 Offsets.swap(OldOffsets);
10515
10516 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
10517 EVT RetVT = OldRetTys[i];
10518 uint64_t Offset = OldOffsets[i];
10519 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
10520 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
10521 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
10522 RetTys.append(NumRegs, RegisterVT);
10523 for (unsigned j = 0; j != NumRegs; ++j)
10524 Offsets.push_back(Offset + j * RegisterVTByteSZ);
10525 }
10526 }
10527
10529 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
10530
10531 bool CanLowerReturn =
10533 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
10534
10535 SDValue DemoteStackSlot;
10536 int DemoteStackIdx = -100;
10537 if (!CanLowerReturn) {
10538 // FIXME: equivalent assert?
10539 // assert(!CS.hasInAllocaArgument() &&
10540 // "sret demotion is incompatible with inalloca");
10541 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
10542 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
10544 DemoteStackIdx =
10545 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
10546 Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
10547 DL.getAllocaAddrSpace());
10548
10549 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
10550 ArgListEntry Entry;
10551 Entry.Node = DemoteStackSlot;
10552 Entry.Ty = StackSlotPtrType;
10553 Entry.IsSExt = false;
10554 Entry.IsZExt = false;
10555 Entry.IsInReg = false;
10556 Entry.IsSRet = true;
10557 Entry.IsNest = false;
10558 Entry.IsByVal = false;
10559 Entry.IsByRef = false;
10560 Entry.IsReturned = false;
10561 Entry.IsSwiftSelf = false;
10562 Entry.IsSwiftAsync = false;
10563 Entry.IsSwiftError = false;
10564 Entry.IsCFGuardTarget = false;
10565 Entry.Alignment = Alignment;
10566 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
10567 CLI.NumFixedArgs += 1;
10568 CLI.getArgs()[0].IndirectType = CLI.RetTy;
10569 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
10570
10571 // sret demotion isn't compatible with tail-calls, since the sret argument
10572 // points into the callers stack frame.
10573 CLI.IsTailCall = false;
10574 } else {
10575 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10576 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
10577 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
10578 ISD::ArgFlagsTy Flags;
10579 if (NeedsRegBlock) {
10580 Flags.setInConsecutiveRegs();
10581 if (I == RetTys.size() - 1)
10582 Flags.setInConsecutiveRegsLast();
10583 }
10584 EVT VT = RetTys[I];
10586 CLI.CallConv, VT);
10587 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10588 CLI.CallConv, VT);
10589 for (unsigned i = 0; i != NumRegs; ++i) {
10590 ISD::InputArg MyFlags;
10591 MyFlags.Flags = Flags;
10592 MyFlags.VT = RegisterVT;
10593 MyFlags.ArgVT = VT;
10594 MyFlags.Used = CLI.IsReturnValueUsed;
10595 if (CLI.RetTy->isPointerTy()) {
10596 MyFlags.Flags.setPointer();
10597 MyFlags.Flags.setPointerAddrSpace(
10598 cast<PointerType>(CLI.RetTy)->getAddressSpace());
10599 }
10600 if (CLI.RetSExt)
10601 MyFlags.Flags.setSExt();
10602 if (CLI.RetZExt)
10603 MyFlags.Flags.setZExt();
10604 if (CLI.IsInReg)
10605 MyFlags.Flags.setInReg();
10606 CLI.Ins.push_back(MyFlags);
10607 }
10608 }
10609 }
10610
10611 // We push in swifterror return as the last element of CLI.Ins.
10612 ArgListTy &Args = CLI.getArgs();
10613 if (supportSwiftError()) {
10614 for (const ArgListEntry &Arg : Args) {
10615 if (Arg.IsSwiftError) {
10616 ISD::InputArg MyFlags;
10617 MyFlags.VT = getPointerTy(DL);
10618 MyFlags.ArgVT = EVT(getPointerTy(DL));
10619 MyFlags.Flags.setSwiftError();
10620 CLI.Ins.push_back(MyFlags);
10621 }
10622 }
10623 }
10624
10625 // Handle all of the outgoing arguments.
10626 CLI.Outs.clear();
10627 CLI.OutVals.clear();
10628 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
10629 SmallVector<EVT, 4> ValueVTs;
10630 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
10631 // FIXME: Split arguments if CLI.IsPostTypeLegalization
10632 Type *FinalType = Args[i].Ty;
10633 if (Args[i].IsByVal)
10634 FinalType = Args[i].IndirectType;
10635 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10636 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
10637 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
10638 ++Value) {
10639 EVT VT = ValueVTs[Value];
10640 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
10641 SDValue Op = SDValue(Args[i].Node.getNode(),
10642 Args[i].Node.getResNo() + Value);
10643 ISD::ArgFlagsTy Flags;
10644
10645 // Certain targets (such as MIPS), may have a different ABI alignment
10646 // for a type depending on the context. Give the target a chance to
10647 // specify the alignment it wants.
10648 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
10649 Flags.setOrigAlign(OriginalAlignment);
10650
10651 if (Args[i].Ty->isPointerTy()) {
10652 Flags.setPointer();
10653 Flags.setPointerAddrSpace(
10654 cast<PointerType>(Args[i].Ty)->getAddressSpace());
10655 }
10656 if (Args[i].IsZExt)
10657 Flags.setZExt();
10658 if (Args[i].IsSExt)
10659 Flags.setSExt();
10660 if (Args[i].IsInReg) {
10661 // If we are using vectorcall calling convention, a structure that is
10662 // passed InReg - is surely an HVA
10664 isa<StructType>(FinalType)) {
10665 // The first value of a structure is marked
10666 if (0 == Value)
10667 Flags.setHvaStart();
10668 Flags.setHva();
10669 }
10670 // Set InReg Flag
10671 Flags.setInReg();
10672 }
10673 if (Args[i].IsSRet)
10674 Flags.setSRet();
10675 if (Args[i].IsSwiftSelf)
10676 Flags.setSwiftSelf();
10677 if (Args[i].IsSwiftAsync)
10678 Flags.setSwiftAsync();
10679 if (Args[i].IsSwiftError)
10680 Flags.setSwiftError();
10681 if (Args[i].IsCFGuardTarget)
10682 Flags.setCFGuardTarget();
10683 if (Args[i].IsByVal)
10684 Flags.setByVal();
10685 if (Args[i].IsByRef)
10686 Flags.setByRef();
10687 if (Args[i].IsPreallocated) {
10688 Flags.setPreallocated();
10689 // Set the byval flag for CCAssignFn callbacks that don't know about
10690 // preallocated. This way we can know how many bytes we should've
10691 // allocated and how many bytes a callee cleanup function will pop. If
10692 // we port preallocated to more targets, we'll have to add custom
10693 // preallocated handling in the various CC lowering callbacks.
10694 Flags.setByVal();
10695 }
10696 if (Args[i].IsInAlloca) {
10697 Flags.setInAlloca();
10698 // Set the byval flag for CCAssignFn callbacks that don't know about
10699 // inalloca. This way we can know how many bytes we should've allocated
10700 // and how many bytes a callee cleanup function will pop. If we port
10701 // inalloca to more targets, we'll have to add custom inalloca handling
10702 // in the various CC lowering callbacks.
10703 Flags.setByVal();
10704 }
10705 Align MemAlign;
10706 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
10707 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
10708 Flags.setByValSize(FrameSize);
10709
10710 // info is not there but there are cases it cannot get right.
10711 if (auto MA = Args[i].Alignment)
10712 MemAlign = *MA;
10713 else
10714 MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
10715 } else if (auto MA = Args[i].Alignment) {
10716 MemAlign = *MA;
10717 } else {
10718 MemAlign = OriginalAlignment;
10719 }
10720 Flags.setMemAlign(MemAlign);
10721 if (Args[i].IsNest)
10722 Flags.setNest();
10723 if (NeedsRegBlock)
10724 Flags.setInConsecutiveRegs();
10725
10727 CLI.CallConv, VT);
10728 unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10729 CLI.CallConv, VT);
10730 SmallVector<SDValue, 4> Parts(NumParts);
10731 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
10732
10733 if (Args[i].IsSExt)
10734 ExtendKind = ISD::SIGN_EXTEND;
10735 else if (Args[i].IsZExt)
10736 ExtendKind = ISD::ZERO_EXTEND;
10737
10738 // Conservatively only handle 'returned' on non-vectors that can be lowered,
10739 // for now.
10740 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
10742 assert((CLI.RetTy == Args[i].Ty ||
10743 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
10745 Args[i].Ty->getPointerAddressSpace())) &&
10746 RetTys.size() == NumValues && "unexpected use of 'returned'");
10747 // Before passing 'returned' to the target lowering code, ensure that
10748 // either the register MVT and the actual EVT are the same size or that
10749 // the return value and argument are extended in the same way; in these
10750 // cases it's safe to pass the argument register value unchanged as the
10751 // return register value (although it's at the target's option whether
10752 // to do so)
10753 // TODO: allow code generation to take advantage of partially preserved
10754 // registers rather than clobbering the entire register when the
10755 // parameter extension method is not compatible with the return
10756 // extension method
10757 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
10758 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
10759 CLI.RetZExt == Args[i].IsZExt))
10760 Flags.setReturned();
10761 }
10762
10763 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
10764 CLI.CallConv, ExtendKind);
10765
10766 for (unsigned j = 0; j != NumParts; ++j) {
10767 // if it isn't first piece, alignment must be 1
10768 // For scalable vectors the scalable part is currently handled
10769 // by individual targets, so we just use the known minimum size here.
10770 ISD::OutputArg MyFlags(
10771 Flags, Parts[j].getValueType().getSimpleVT(), VT,
10772 i < CLI.NumFixedArgs, i,
10773 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
10774 if (NumParts > 1 && j == 0)
10775 MyFlags.Flags.setSplit();
10776 else if (j != 0) {
10777 MyFlags.Flags.setOrigAlign(Align(1));
10778 if (j == NumParts - 1)
10779 MyFlags.Flags.setSplitEnd();
10780 }
10781
10782 CLI.Outs.push_back(MyFlags);
10783 CLI.OutVals.push_back(Parts[j]);
10784 }
10785
10786 if (NeedsRegBlock && Value == NumValues - 1)
10787 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
10788 }
10789 }
10790
10792 CLI.Chain = LowerCall(CLI, InVals);
10793
10794 // Update CLI.InVals to use outside of this function.
10795 CLI.InVals = InVals;
10796
10797 // Verify that the target's LowerCall behaved as expected.
10798 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
10799 "LowerCall didn't return a valid chain!");
10800 assert((!CLI.IsTailCall || InVals.empty()) &&
10801 "LowerCall emitted a return value for a tail call!");
10802 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
10803 "LowerCall didn't emit the correct number of values!");
10804
10805 // For a tail call, the return value is merely live-out and there aren't
10806 // any nodes in the DAG representing it. Return a special value to
10807 // indicate that a tail call has been emitted and no more Instructions
10808 // should be processed in the current block.
10809 if (CLI.IsTailCall) {
10810 CLI.DAG.setRoot(CLI.Chain);
10811 return std::make_pair(SDValue(), SDValue());
10812 }
10813
10814#ifndef NDEBUG
10815 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
10816 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
10817 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
10818 "LowerCall emitted a value with the wrong type!");
10819 }
10820#endif
10821
10822 SmallVector<SDValue, 4> ReturnValues;
10823 if (!CanLowerReturn) {
10824 // The instruction result is the result of loading from the
10825 // hidden sret parameter.
10827 Type *PtrRetTy =
10828 PointerType::get(OrigRetTy->getContext(), DL.getAllocaAddrSpace());
10829
10830 ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
10831 assert(PVTs.size() == 1 && "Pointers should fit in one register");
10832 EVT PtrVT = PVTs[0];
10833
10834 unsigned NumValues = RetTys.size();
10835 ReturnValues.resize(NumValues);
10836 SmallVector<SDValue, 4> Chains(NumValues);
10837
10838 // An aggregate return value cannot wrap around the address space, so
10839 // offsets to its parts don't wrap either.
10840 SDNodeFlags Flags;
10841 Flags.setNoUnsignedWrap(true);
10842
10844 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
10845 for (unsigned i = 0; i < NumValues; ++i) {
10846 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
10847 CLI.DAG.getConstant(Offsets[i], CLI.DL,
10848 PtrVT), Flags);
10849 SDValue L = CLI.DAG.getLoad(
10850 RetTys[i], CLI.DL, CLI.Chain, Add,
10852 DemoteStackIdx, Offsets[i]),
10853 HiddenSRetAlign);
10854 ReturnValues[i] = L;
10855 Chains[i] = L.getValue(1);
10856 }
10857
10858 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
10859 } else {
10860 // Collect the legal value parts into potentially illegal values
10861 // that correspond to the original function's return values.
10862 std::optional<ISD::NodeType> AssertOp;
10863 if (CLI.RetSExt)
10864 AssertOp = ISD::AssertSext;
10865 else if (CLI.RetZExt)
10866 AssertOp = ISD::AssertZext;
10867 unsigned CurReg = 0;
10868 for (EVT VT : RetTys) {
10870 CLI.CallConv, VT);
10871 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10872 CLI.CallConv, VT);
10873
10874 ReturnValues.push_back(getCopyFromParts(
10875 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
10876 CLI.Chain, CLI.CallConv, AssertOp));
10877 CurReg += NumRegs;
10878 }
10879
10880 // For a function returning void, there is no return value. We can't create
10881 // such a node, so we just return a null return value in that case. In
10882 // that case, nothing will actually look at the value.
10883 if (ReturnValues.empty())
10884 return std::make_pair(SDValue(), CLI.Chain);
10885 }
10886
10887 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
10888 CLI.DAG.getVTList(RetTys), ReturnValues);
10889 return std::make_pair(Res, CLI.Chain);
10890}
10891
10892/// Places new result values for the node in Results (their number
10893/// and types must exactly match those of the original return values of
10894/// the node), or leaves Results empty, which indicates that the node is not
10895/// to be custom lowered after all.
10898 SelectionDAG &DAG) const {
10899 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
10900
10901 if (!Res.getNode())
10902 return;
10903
10904 // If the original node has one result, take the return value from
10905 // LowerOperation as is. It might not be result number 0.
10906 if (N->getNumValues() == 1) {
10907 Results.push_back(Res);
10908 return;
10909 }
10910
10911 // If the original node has multiple results, then the return node should
10912 // have the same number of results.
10913 assert((N->getNumValues() == Res->getNumValues()) &&
10914 "Lowering returned the wrong number of results!");
10915
10916 // Places new result values base on N result number.
10917 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
10918 Results.push_back(Res.getValue(I));
10919}
10920
10922 llvm_unreachable("LowerOperation not implemented for this target!");
10923}
10924
10926 unsigned Reg,
10927 ISD::NodeType ExtendType) {
10929 assert((Op.getOpcode() != ISD::CopyFromReg ||
10930 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
10931 "Copy from a reg to the same reg!");
10932 assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
10933
10935 // If this is an InlineAsm we have to match the registers required, not the
10936 // notional registers required by the type.
10937
10938 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
10939 std::nullopt); // This is not an ABI copy.
10940 SDValue Chain = DAG.getEntryNode();
10941
10942 if (ExtendType == ISD::ANY_EXTEND) {
10943 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
10944 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
10945 ExtendType = PreferredExtendIt->second;
10946 }
10947 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
10948 PendingExports.push_back(Chain);
10949}
10950
10952
10953/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10954/// entry block, return true. This includes arguments used by switches, since
10955/// the switch may expand into multiple basic blocks.
10956static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
10957 // With FastISel active, we may be splitting blocks, so force creation
10958 // of virtual registers for all non-dead arguments.
10959 if (FastISel)
10960 return A->use_empty();
10961
10962 const BasicBlock &Entry = A->getParent()->front();
10963 for (const User *U : A->users())
10964 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
10965 return false; // Use not in entry block.
10966
10967 return true;
10968}
10969
10971 DenseMap<const Argument *,
10972 std::pair<const AllocaInst *, const StoreInst *>>;
10973
10974/// Scan the entry block of the function in FuncInfo for arguments that look
10975/// like copies into a local alloca. Record any copied arguments in
10976/// ArgCopyElisionCandidates.
10977static void
10979 FunctionLoweringInfo *FuncInfo,
10980 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
10981 // Record the state of every static alloca used in the entry block. Argument
10982 // allocas are all used in the entry block, so we need approximately as many
10983 // entries as we have arguments.
10984 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
10986 unsigned NumArgs = FuncInfo->Fn->arg_size();
10987 StaticAllocas.reserve(NumArgs * 2);
10988
10989 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
10990 if (!V)
10991 return nullptr;
10992 V = V->stripPointerCasts();
10993 const auto *AI = dyn_cast<AllocaInst>(V);
10994 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
10995 return nullptr;
10996 auto Iter = StaticAllocas.insert({AI, Unknown});
10997 return &Iter.first->second;
10998 };
10999
11000 // Look for stores of arguments to static allocas. Look through bitcasts and
11001 // GEPs to handle type coercions, as long as the alloca is fully initialized
11002 // by the store. Any non-store use of an alloca escapes it and any subsequent
11003 // unanalyzed store might write it.
11004 // FIXME: Handle structs initialized with multiple stores.
11005 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11006 // Look for stores, and handle non-store uses conservatively.
11007 const auto *SI = dyn_cast<StoreInst>(&I);
11008 if (!SI) {
11009 // We will look through cast uses, so ignore them completely.
11010 if (I.isCast())
11011 continue;
11012 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11013 // to allocas.
11014 if (I.isDebugOrPseudoInst())
11015 continue;
11016 // This is an unknown instruction. Assume it escapes or writes to all
11017 // static alloca operands.
11018 for (const Use &U : I.operands()) {
11019 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11020 *Info = StaticAllocaInfo::Clobbered;
11021 }
11022 continue;
11023 }
11024
11025 // If the stored value is a static alloca, mark it as escaped.
11026 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11027 *Info = StaticAllocaInfo::Clobbered;
11028
11029 // Check if the destination is a static alloca.
11030 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11031 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11032 if (!Info)
11033 continue;
11034 const AllocaInst *AI = cast<AllocaInst>(Dst);
11035
11036 // Skip allocas that have been initialized or clobbered.
11037 if (*Info != StaticAllocaInfo::Unknown)
11038 continue;
11039
11040 // Check if the stored value is an argument, and that this store fully
11041 // initializes the alloca.
11042 // If the argument type has padding bits we can't directly forward a pointer
11043 // as the upper bits may contain garbage.
11044 // Don't elide copies from the same argument twice.
11045 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11046 const auto *Arg = dyn_cast<Argument>(Val);
11047 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11048 Arg->getType()->isEmptyTy() ||
11049 DL.getTypeStoreSize(Arg->getType()) !=
11050 DL.getTypeAllocSize(AI->getAllocatedType()) ||
11051 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
11052 ArgCopyElisionCandidates.count(Arg)) {
11053 *Info = StaticAllocaInfo::Clobbered;
11054 continue;
11055 }
11056
11057 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11058 << '\n');
11059
11060 // Mark this alloca and store for argument copy elision.
11061 *Info = StaticAllocaInfo::Elidable;
11062 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
11063
11064 // Stop scanning if we've seen all arguments. This will happen early in -O0
11065 // builds, which is useful, because -O0 builds have large entry blocks and
11066 // many allocas.
11067 if (ArgCopyElisionCandidates.size() == NumArgs)
11068 break;
11069 }
11070}
11071
11072/// Try to elide argument copies from memory into a local alloca. Succeeds if
11073/// ArgVal is a load from a suitable fixed stack object.
11076 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11077 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11078 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11079 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11080 // Check if this is a load from a fixed stack object.
11081 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
11082 if (!LNode)
11083 return;
11084 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
11085 if (!FINode)
11086 return;
11087
11088 // Check that the fixed stack object is the right size and alignment.
11089 // Look at the alignment that the user wrote on the alloca instead of looking
11090 // at the stack object.
11091 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
11092 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11093 const AllocaInst *AI = ArgCopyIter->second.first;
11094 int FixedIndex = FINode->getIndex();
11095 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11096 int OldIndex = AllocaIndex;
11097 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11098 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
11099 LLVM_DEBUG(
11100 dbgs() << " argument copy elision failed due to bad fixed stack "
11101 "object size\n");
11102 return;
11103 }
11104 Align RequiredAlignment = AI->getAlign();
11105 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
11106 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11107 "greater than stack argument alignment ("
11108 << DebugStr(RequiredAlignment) << " vs "
11109 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11110 return;
11111 }
11112
11113 // Perform the elision. Delete the old stack object and replace its only use
11114 // in the variable info map. Mark the stack object as mutable.
11115 LLVM_DEBUG({
11116 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11117 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11118 << '\n';
11119 });
11120 MFI.RemoveStackObject(OldIndex);
11121 MFI.setIsImmutableObjectIndex(FixedIndex, false);
11122 AllocaIndex = FixedIndex;
11123 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
11124 for (SDValue ArgVal : ArgVals)
11125 Chains.push_back(ArgVal.getValue(1));
11126
11127 // Avoid emitting code for the store implementing the copy.
11128 const StoreInst *SI = ArgCopyIter->second.second;
11129 ElidedArgCopyInstrs.insert(SI);
11130
11131 // Check for uses of the argument again so that we can avoid exporting ArgVal
11132 // if it is't used by anything other than the store.
11133 for (const Value *U : Arg.users()) {
11134 if (U != SI) {
11135 ArgHasUses = true;
11136 break;
11137 }
11138 }
11139}
11140
11141void SelectionDAGISel::LowerArguments(const Function &F) {
11142 SelectionDAG &DAG = SDB->DAG;
11143 SDLoc dl = SDB->getCurSDLoc();
11144 const DataLayout &DL = DAG.getDataLayout();
11146
11147 // In Naked functions we aren't going to save any registers.
11148 if (F.hasFnAttribute(Attribute::Naked))
11149 return;
11150
11151 if (!FuncInfo->CanLowerReturn) {
11152 // Put in an sret pointer parameter before all the other parameters.
11153 SmallVector<EVT, 1> ValueVTs;
11155 PointerType::get(F.getContext(),
11157 ValueVTs);
11158
11159 // NOTE: Assuming that a pointer will never break down to more than one VT
11160 // or one register.
11162 Flags.setSRet();
11163 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
11164 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
11166 Ins.push_back(RetArg);
11167 }
11168
11169 // Look for stores of arguments to static allocas. Mark such arguments with a
11170 // flag to ask the target to give us the memory location of that argument if
11171 // available.
11172 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11174 ArgCopyElisionCandidates);
11175
11176 // Set up the incoming argument description vector.
11177 for (const Argument &Arg : F.args()) {
11178 unsigned ArgNo = Arg.getArgNo();
11179 SmallVector<EVT, 4> ValueVTs;
11180 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11181 bool isArgValueUsed = !Arg.use_empty();
11182 unsigned PartBase = 0;
11183 Type *FinalType = Arg.getType();
11184 if (Arg.hasAttribute(Attribute::ByVal))
11185 FinalType = Arg.getParamByValType();
11186 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11187 FinalType, F.getCallingConv(), F.isVarArg(), DL);
11188 for (unsigned Value = 0, NumValues = ValueVTs.size();
11189 Value != NumValues; ++Value) {
11190 EVT VT = ValueVTs[Value];
11191 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
11193
11194
11195 if (Arg.getType()->isPointerTy()) {
11196 Flags.setPointer();
11197 Flags.setPointerAddrSpace(
11198 cast<PointerType>(Arg.getType())->getAddressSpace());
11199 }
11200 if (Arg.hasAttribute(Attribute::ZExt))
11201 Flags.setZExt();
11202 if (Arg.hasAttribute(Attribute::SExt))
11203 Flags.setSExt();
11204 if (Arg.hasAttribute(Attribute::InReg)) {
11205 // If we are using vectorcall calling convention, a structure that is
11206 // passed InReg - is surely an HVA
11207 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11208 isa<StructType>(Arg.getType())) {
11209 // The first value of a structure is marked
11210 if (0 == Value)
11211 Flags.setHvaStart();
11212 Flags.setHva();
11213 }
11214 // Set InReg Flag
11215 Flags.setInReg();
11216 }
11217 if (Arg.hasAttribute(Attribute::StructRet))
11218 Flags.setSRet();
11219 if (Arg.hasAttribute(Attribute::SwiftSelf))
11220 Flags.setSwiftSelf();
11221 if (Arg.hasAttribute(Attribute::SwiftAsync))
11222 Flags.setSwiftAsync();
11223 if (Arg.hasAttribute(Attribute::SwiftError))
11224 Flags.setSwiftError();
11225 if (Arg.hasAttribute(Attribute::ByVal))
11226 Flags.setByVal();
11227 if (Arg.hasAttribute(Attribute::ByRef))
11228 Flags.setByRef();
11229 if (Arg.hasAttribute(Attribute::InAlloca)) {
11230 Flags.setInAlloca();
11231 // Set the byval flag for CCAssignFn callbacks that don't know about
11232 // inalloca. This way we can know how many bytes we should've allocated
11233 // and how many bytes a callee cleanup function will pop. If we port
11234 // inalloca to more targets, we'll have to add custom inalloca handling
11235 // in the various CC lowering callbacks.
11236 Flags.setByVal();
11237 }
11238 if (Arg.hasAttribute(Attribute::Preallocated)) {
11239 Flags.setPreallocated();
11240 // Set the byval flag for CCAssignFn callbacks that don't know about
11241 // preallocated. This way we can know how many bytes we should've
11242 // allocated and how many bytes a callee cleanup function will pop. If
11243 // we port preallocated to more targets, we'll have to add custom
11244 // preallocated handling in the various CC lowering callbacks.
11245 Flags.setByVal();
11246 }
11247
11248 // Certain targets (such as MIPS), may have a different ABI alignment
11249 // for a type depending on the context. Give the target a chance to
11250 // specify the alignment it wants.
11251 const Align OriginalAlignment(
11253 Flags.setOrigAlign(OriginalAlignment);
11254
11255 Align MemAlign;
11256 Type *ArgMemTy = nullptr;
11257 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11258 Flags.isByRef()) {
11259 if (!ArgMemTy)
11260 ArgMemTy = Arg.getPointeeInMemoryValueType();
11261
11262 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
11263
11264 // For in-memory arguments, size and alignment should be passed from FE.
11265 // BE will guess if this info is not there but there are cases it cannot
11266 // get right.
11267 if (auto ParamAlign = Arg.getParamStackAlign())
11268 MemAlign = *ParamAlign;
11269 else if ((ParamAlign = Arg.getParamAlign()))
11270 MemAlign = *ParamAlign;
11271 else
11272 MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
11273 if (Flags.isByRef())
11274 Flags.setByRefSize(MemSize);
11275 else
11276 Flags.setByValSize(MemSize);
11277 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
11278 MemAlign = *ParamAlign;
11279 } else {
11280 MemAlign = OriginalAlignment;
11281 }
11282 Flags.setMemAlign(MemAlign);
11283
11284 if (Arg.hasAttribute(Attribute::Nest))
11285 Flags.setNest();
11286 if (NeedsRegBlock)
11287 Flags.setInConsecutiveRegs();
11288 if (ArgCopyElisionCandidates.count(&Arg))
11289 Flags.setCopyElisionCandidate();
11290 if (Arg.hasAttribute(Attribute::Returned))
11291 Flags.setReturned();
11292
11294 *CurDAG->getContext(), F.getCallingConv(), VT);
11295 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
11296 *CurDAG->getContext(), F.getCallingConv(), VT);
11297 for (unsigned i = 0; i != NumRegs; ++i) {
11298 // For scalable vectors, use the minimum size; individual targets
11299 // are responsible for handling scalable vector arguments and
11300 // return values.
11301 ISD::InputArg MyFlags(
11302 Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
11303 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
11304 if (NumRegs > 1 && i == 0)
11305 MyFlags.Flags.setSplit();
11306 // if it isn't first piece, alignment must be 1
11307 else if (i > 0) {
11308 MyFlags.Flags.setOrigAlign(Align(1));
11309 if (i == NumRegs - 1)
11310 MyFlags.Flags.setSplitEnd();
11311 }
11312 Ins.push_back(MyFlags);
11313 }
11314 if (NeedsRegBlock && Value == NumValues - 1)
11315 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11316 PartBase += VT.getStoreSize().getKnownMinValue();
11317 }
11318 }
11319
11320 // Call the target to set up the argument values.
11322 SDValue NewRoot = TLI->LowerFormalArguments(
11323 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
11324
11325 // Verify that the target's LowerFormalArguments behaved as expected.
11326 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
11327 "LowerFormalArguments didn't return a valid chain!");
11328 assert(InVals.size() == Ins.size() &&
11329 "LowerFormalArguments didn't emit the correct number of values!");
11330 LLVM_DEBUG({
11331 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
11332 assert(InVals[i].getNode() &&
11333 "LowerFormalArguments emitted a null value!");
11334 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11335 "LowerFormalArguments emitted a value with the wrong type!");
11336 }
11337 });
11338
11339 // Update the DAG with the new chain value resulting from argument lowering.
11340 DAG.setRoot(NewRoot);
11341
11342 // Set up the argument values.
11343 unsigned i = 0;
11344 if (!FuncInfo->CanLowerReturn) {
11345 // Create a virtual register for the sret pointer, and put in a copy
11346 // from the sret argument into it.
11347 SmallVector<EVT, 1> ValueVTs;
11349 PointerType::get(F.getContext(),
11351 ValueVTs);
11352 MVT VT = ValueVTs[0].getSimpleVT();
11353 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
11354 std::optional<ISD::NodeType> AssertOp;
11355 SDValue ArgValue =
11356 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
11357 F.getCallingConv(), AssertOp);
11358
11359 MachineFunction& MF = SDB->DAG.getMachineFunction();
11361 Register SRetReg =
11362 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
11363 FuncInfo->DemoteRegister = SRetReg;
11364 NewRoot =
11365 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
11366 DAG.setRoot(NewRoot);
11367
11368 // i indexes lowered arguments. Bump it past the hidden sret argument.
11369 ++i;
11370 }
11371
11373 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
11374 for (const Argument &Arg : F.args()) {
11375 SmallVector<SDValue, 4> ArgValues;
11376 SmallVector<EVT, 4> ValueVTs;
11377 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11378 unsigned NumValues = ValueVTs.size();
11379 if (NumValues == 0)
11380 continue;
11381
11382 bool ArgHasUses = !Arg.use_empty();
11383
11384 // Elide the copying store if the target loaded this argument from a
11385 // suitable fixed stack object.
11386 if (Ins[i].Flags.isCopyElisionCandidate()) {
11387 unsigned NumParts = 0;
11388 for (EVT VT : ValueVTs)
11390 F.getCallingConv(), VT);
11391
11392 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
11393 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
11394 ArrayRef(&InVals[i], NumParts), ArgHasUses);
11395 }
11396
11397 // If this argument is unused then remember its value. It is used to generate
11398 // debugging information.
11399 bool isSwiftErrorArg =
11401 Arg.hasAttribute(Attribute::SwiftError);
11402 if (!ArgHasUses && !isSwiftErrorArg) {
11403 SDB->setUnusedArgValue(&Arg, InVals[i]);
11404
11405 // Also remember any frame index for use in FastISel.
11406 if (FrameIndexSDNode *FI =
11407 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
11408 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11409 }
11410
11411 for (unsigned Val = 0; Val != NumValues; ++Val) {
11412 EVT VT = ValueVTs[Val];
11414 F.getCallingConv(), VT);
11415 unsigned NumParts = TLI->getNumRegistersForCallingConv(
11416 *CurDAG->getContext(), F.getCallingConv(), VT);
11417
11418 // Even an apparent 'unused' swifterror argument needs to be returned. So
11419 // we do generate a copy for it that can be used on return from the
11420 // function.
11421 if (ArgHasUses || isSwiftErrorArg) {
11422 std::optional<ISD::NodeType> AssertOp;
11423 if (Arg.hasAttribute(Attribute::SExt))
11424 AssertOp = ISD::AssertSext;
11425 else if (Arg.hasAttribute(Attribute::ZExt))
11426 AssertOp = ISD::AssertZext;
11427
11428 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
11429 PartVT, VT, nullptr, NewRoot,
11430 F.getCallingConv(), AssertOp));
11431 }
11432
11433 i += NumParts;
11434 }
11435
11436 // We don't need to do anything else for unused arguments.
11437 if (ArgValues.empty())
11438 continue;
11439
11440 // Note down frame index.
11441 if (FrameIndexSDNode *FI =
11442 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
11443 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11444
11445 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
11446 SDB->getCurSDLoc());
11447
11448 SDB->setValue(&Arg, Res);
11450 // We want to associate the argument with the frame index, among
11451 // involved operands, that correspond to the lowest address. The
11452 // getCopyFromParts function, called earlier, is swapping the order of
11453 // the operands to BUILD_PAIR depending on endianness. The result of
11454 // that swapping is that the least significant bits of the argument will
11455 // be in the first operand of the BUILD_PAIR node, and the most
11456 // significant bits will be in the second operand.
11457 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
11458 if (LoadSDNode *LNode =
11459 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
11460 if (FrameIndexSDNode *FI =
11461 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
11462 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11463 }
11464
11465 // Analyses past this point are naive and don't expect an assertion.
11466 if (Res.getOpcode() == ISD::AssertZext)
11467 Res = Res.getOperand(0);
11468
11469 // Update the SwiftErrorVRegDefMap.
11470 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
11471 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11474 Reg);
11475 }
11476
11477 // If this argument is live outside of the entry block, insert a copy from
11478 // wherever we got it to the vreg that other BB's will reference it as.
11479 if (Res.getOpcode() == ISD::CopyFromReg) {
11480 // If we can, though, try to skip creating an unnecessary vreg.
11481 // FIXME: This isn't very clean... it would be nice to make this more
11482 // general.
11483 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11484 if (Register::isVirtualRegister(Reg)) {
11485 FuncInfo->ValueMap[&Arg] = Reg;
11486 continue;
11487 }
11488 }
11490 FuncInfo->InitializeRegForValue(&Arg);
11491 SDB->CopyToExportRegsIfNeeded(&Arg);
11492 }
11493 }
11494
11495 if (!Chains.empty()) {
11496 Chains.push_back(NewRoot);
11497 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
11498 }
11499
11500 DAG.setRoot(NewRoot);
11501
11502 assert(i == InVals.size() && "Argument register count mismatch!");
11503
11504 // If any argument copy elisions occurred and we have debug info, update the
11505 // stale frame indices used in the dbg.declare variable info table.
11506 if (!ArgCopyElisionFrameIndexMap.empty()) {
11509 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
11510 if (I != ArgCopyElisionFrameIndexMap.end())
11511 VI.updateStackSlot(I->second);
11512 }
11513 }
11514
11515 // Finally, if the target has anything special to do, allow it to do so.
11517}
11518
11519/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
11520/// ensure constants are generated when needed. Remember the virtual registers
11521/// that need to be added to the Machine PHI nodes as input. We cannot just
11522/// directly add them, because expansion might result in multiple MBB's for one
11523/// BB. As such, the start of the BB might correspond to a different MBB than
11524/// the end.
11525void
11526SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
11528
11530
11531 // Check PHI nodes in successors that expect a value to be available from this
11532 // block.
11533 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
11534 if (!isa<PHINode>(SuccBB->begin())) continue;
11535 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
11536
11537 // If this terminator has multiple identical successors (common for
11538 // switches), only handle each succ once.
11539 if (!SuccsHandled.insert(SuccMBB).second)
11540 continue;
11541
11543
11544 // At this point we know that there is a 1-1 correspondence between LLVM PHI
11545 // nodes and Machine PHI nodes, but the incoming operands have not been
11546 // emitted yet.
11547 for (const PHINode &PN : SuccBB->phis()) {
11548 // Ignore dead phi's.
11549 if (PN.use_empty())
11550 continue;
11551
11552 // Skip empty types
11553 if (PN.getType()->isEmptyTy())
11554 continue;
11555
11556 unsigned Reg;
11557 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
11558
11559 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
11560 unsigned &RegOut = ConstantsOut[C];
11561 if (RegOut == 0) {
11562 RegOut = FuncInfo.CreateRegs(C);
11563 // We need to zero/sign extend ConstantInt phi operands to match
11564 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
11565 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
11566 if (auto *CI = dyn_cast<ConstantInt>(C))
11567 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
11569 CopyValueToVirtualRegister(C, RegOut, ExtendType);
11570 }
11571 Reg = RegOut;
11572 } else {
11574 FuncInfo.ValueMap.find(PHIOp);
11575 if (I != FuncInfo.ValueMap.end())
11576 Reg = I->second;
11577 else {
11578 assert(isa<AllocaInst>(PHIOp) &&
11579 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
11580 "Didn't codegen value into a register!??");
11581 Reg = FuncInfo.CreateRegs(PHIOp);
11582 CopyValueToVirtualRegister(PHIOp, Reg);
11583 }
11584 }
11585
11586 // Remember that this register needs to added to the machine PHI node as
11587 // the input for this MBB.
11588 SmallVector<EVT, 4> ValueVTs;
11589 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
11590 for (EVT VT : ValueVTs) {
11591 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
11592 for (unsigned i = 0; i != NumRegisters; ++i)
11593 FuncInfo.PHINodesToUpdate.push_back(
11594 std::make_pair(&*MBBI++, Reg + i));
11595 Reg += NumRegisters;
11596 }
11597 }
11598 }
11599
11600 ConstantsOut.clear();
11601}
11602
11603MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
11605 if (++I == FuncInfo.MF->end())
11606 return nullptr;
11607 return &*I;
11608}
11609
11610/// During lowering new call nodes can be created (such as memset, etc.).
11611/// Those will become new roots of the current DAG, but complications arise
11612/// when they are tail calls. In such cases, the call lowering will update
11613/// the root, but the builder still needs to know that a tail call has been
11614/// lowered in order to avoid generating an additional return.
11615void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
11616 // If the node is null, we do have a tail call.
11617 if (MaybeTC.getNode() != nullptr)
11618 DAG.setRoot(MaybeTC);
11619 else
11620 HasTailCall = true;
11621}
11622
11623void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
11624 MachineBasicBlock *SwitchMBB,
11625 MachineBasicBlock *DefaultMBB) {
11626 MachineFunction *CurMF = FuncInfo.MF;
11627 MachineBasicBlock *NextMBB = nullptr;
11629 if (++BBI != FuncInfo.MF->end())
11630 NextMBB = &*BBI;
11631
11632 unsigned Size = W.LastCluster - W.FirstCluster + 1;
11633
11635
11636 if (Size == 2 && W.MBB == SwitchMBB) {
11637 // If any two of the cases has the same destination, and if one value
11638 // is the same as the other, but has one bit unset that the other has set,
11639 // use bit manipulation to do two compares at once. For example:
11640 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
11641 // TODO: This could be extended to merge any 2 cases in switches with 3
11642 // cases.
11643 // TODO: Handle cases where W.CaseBB != SwitchBB.
11644 CaseCluster &Small = *W.FirstCluster;
11645 CaseCluster &Big = *W.LastCluster;
11646
11647 if (Small.Low == Small.High && Big.Low == Big.High &&
11648 Small.MBB == Big.MBB) {
11649 const APInt &SmallValue = Small.Low->getValue();
11650 const APInt &BigValue = Big.Low->getValue();
11651
11652 // Check that there is only one bit different.
11653 APInt CommonBit = BigValue ^ SmallValue;
11654 if (CommonBit.isPowerOf2()) {
11655 SDValue CondLHS = getValue(Cond);
11656 EVT VT = CondLHS.getValueType();
11657 SDLoc DL = getCurSDLoc();
11658
11659 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
11660 DAG.getConstant(CommonBit, DL, VT));
11662 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
11663 ISD::SETEQ);
11664
11665 // Update successor info.
11666 // Both Small and Big will jump to Small.BB, so we sum up the
11667 // probabilities.
11668 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
11669 if (BPI)
11670 addSuccessorWithProb(
11671 SwitchMBB, DefaultMBB,
11672 // The default destination is the first successor in IR.
11673 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
11674 else
11675 addSuccessorWithProb(SwitchMBB, DefaultMBB);
11676
11677 // Insert the true branch.
11678 SDValue BrCond =
11679 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
11680 DAG.getBasicBlock(Small.MBB));
11681 // Insert the false branch.
11682 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
11683 DAG.getBasicBlock(DefaultMBB));
11684
11685 DAG.setRoot(BrCond);
11686 return;
11687 }
11688 }
11689 }
11690
11691 if (TM.getOptLevel() != CodeGenOptLevel::None) {
11692 // Here, we order cases by probability so the most likely case will be
11693 // checked first. However, two clusters can have the same probability in
11694 // which case their relative ordering is non-deterministic. So we use Low
11695 // as a tie-breaker as clusters are guaranteed to never overlap.
11696 llvm::sort(W.FirstCluster, W.LastCluster + 1,
11697 [](const CaseCluster &a, const CaseCluster &b) {
11698 return a.Prob != b.Prob ?
11699 a.Prob > b.Prob :
11700 a.Low->getValue().slt(b.Low->getValue());
11701 });
11702
11703 // Rearrange the case blocks so that the last one falls through if possible
11704 // without changing the order of probabilities.
11705 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
11706 --I;
11707 if (I->Prob > W.LastCluster->Prob)
11708 break;
11709 if (I->Kind == CC_Range && I->MBB == NextMBB) {
11710 std::swap(*I, *W.LastCluster);
11711 break;
11712 }
11713 }
11714 }
11715
11716 // Compute total probability.
11717 BranchProbability DefaultProb = W.DefaultProb;
11718 BranchProbability UnhandledProbs = DefaultProb;
11719 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
11720 UnhandledProbs += I->Prob;
11721
11722 MachineBasicBlock *CurMBB = W.MBB;
11723 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
11724 bool FallthroughUnreachable = false;
11725 MachineBasicBlock *Fallthrough;
11726 if (I == W.LastCluster) {
11727 // For the last cluster, fall through to the default destination.
11728 Fallthrough = DefaultMBB;
11729 FallthroughUnreachable = isa<UnreachableInst>(
11730 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
11731 } else {
11732 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
11733 CurMF->insert(BBI, Fallthrough);
11734 // Put Cond in a virtual register to make it available from the new blocks.
11736 }
11737 UnhandledProbs -= I->Prob;
11738
11739 switch (I->Kind) {
11740 case CC_JumpTable: {
11741 // FIXME: Optimize away range check based on pivot comparisons.
11742 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
11743 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
11744
11745 // The jump block hasn't been inserted yet; insert it here.
11746 MachineBasicBlock *JumpMBB = JT->MBB;
11747 CurMF->insert(BBI, JumpMBB);
11748
11749 auto JumpProb = I->Prob;
11750 auto FallthroughProb = UnhandledProbs;
11751
11752 // If the default statement is a target of the jump table, we evenly
11753 // distribute the default probability to successors of CurMBB. Also
11754 // update the probability on the edge from JumpMBB to Fallthrough.
11755 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
11756 SE = JumpMBB->succ_end();
11757 SI != SE; ++SI) {
11758 if (*SI == DefaultMBB) {
11759 JumpProb += DefaultProb / 2;
11760 FallthroughProb -= DefaultProb / 2;
11761 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
11762 JumpMBB->normalizeSuccProbs();
11763 break;
11764 }
11765 }
11766
11767 // If the default clause is unreachable, propagate that knowledge into
11768 // JTH->FallthroughUnreachable which will use it to suppress the range
11769 // check.
11770 //
11771 // However, don't do this if we're doing branch target enforcement,
11772 // because a table branch _without_ a range check can be a tempting JOP
11773 // gadget - out-of-bounds inputs that are impossible in correct
11774 // execution become possible again if an attacker can influence the
11775 // control flow. So if an attacker doesn't already have a BTI bypass
11776 // available, we don't want them to be able to get one out of this
11777 // table branch.
11778 if (FallthroughUnreachable) {
11779 Function &CurFunc = CurMF->getFunction();
11780 bool HasBranchTargetEnforcement = false;
11781 if (CurFunc.hasFnAttribute("branch-target-enforcement")) {
11782 HasBranchTargetEnforcement =
11783 CurFunc.getFnAttribute("branch-target-enforcement")
11784 .getValueAsBool();
11785 } else {
11786 HasBranchTargetEnforcement =
11787 CurMF->getMMI().getModule()->getModuleFlag(
11788 "branch-target-enforcement");
11789 }
11790 if (!HasBranchTargetEnforcement)
11791 JTH->FallthroughUnreachable = true;
11792 }
11793
11794 if (!JTH->FallthroughUnreachable)
11795 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
11796 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
11797 CurMBB->normalizeSuccProbs();
11798
11799 // The jump table header will be inserted in our current block, do the
11800 // range check, and fall through to our fallthrough block.
11801 JTH->HeaderBB = CurMBB;
11802 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
11803
11804 // If we're in the right place, emit the jump table header right now.
11805 if (CurMBB == SwitchMBB) {
11806 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
11807 JTH->Emitted = true;
11808 }
11809 break;
11810 }
11811 case CC_BitTests: {
11812 // FIXME: Optimize away range check based on pivot comparisons.
11813 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
11814
11815 // The bit test blocks haven't been inserted yet; insert them here.
11816 for (BitTestCase &BTC : BTB->Cases)
11817 CurMF->insert(BBI, BTC.ThisBB);
11818
11819 // Fill in fields of the BitTestBlock.
11820 BTB->Parent = CurMBB;
11821 BTB->Default = Fallthrough;
11822
11823 BTB->DefaultProb = UnhandledProbs;
11824 // If the cases in bit test don't form a contiguous range, we evenly
11825 // distribute the probability on the edge to Fallthrough to two
11826 // successors of CurMBB.
11827 if (!BTB->ContiguousRange) {
11828 BTB->Prob += DefaultProb / 2;
11829 BTB->DefaultProb -= DefaultProb / 2;
11830 }
11831
11832 if (FallthroughUnreachable)
11833 BTB->FallthroughUnreachable = true;
11834
11835 // If we're in the right place, emit the bit test header right now.
11836 if (CurMBB == SwitchMBB) {
11837 visitBitTestHeader(*BTB, SwitchMBB);
11838 BTB->Emitted = true;
11839 }
11840 break;
11841 }
11842 case CC_Range: {
11843 const Value *RHS, *LHS, *MHS;
11845 if (I->Low == I->High) {
11846 // Check Cond == I->Low.
11847 CC = ISD::SETEQ;
11848 LHS = Cond;
11849 RHS=I->Low;
11850 MHS = nullptr;
11851 } else {
11852 // Check I->Low <= Cond <= I->High.
11853 CC = ISD::SETLE;
11854 LHS = I->Low;
11855 MHS = Cond;
11856 RHS = I->High;
11857 }
11858
11859 // If Fallthrough is unreachable, fold away the comparison.
11860 if (FallthroughUnreachable)
11861 CC = ISD::SETTRUE;
11862
11863 // The false probability is the sum of all unhandled cases.
11864 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
11865 getCurSDLoc(), I->Prob, UnhandledProbs);
11866
11867 if (CurMBB == SwitchMBB)
11868 visitSwitchCase(CB, SwitchMBB);
11869 else
11870 SL->SwitchCases.push_back(CB);
11871
11872 break;
11873 }
11874 }
11875 CurMBB = Fallthrough;
11876 }
11877}
11878
11879void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
11880 const SwitchWorkListItem &W,
11881 Value *Cond,
11882 MachineBasicBlock *SwitchMBB) {
11883 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
11884 "Clusters not sorted?");
11885 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
11886
11887 auto [LastLeft, FirstRight, LeftProb, RightProb] =
11888 SL->computeSplitWorkItemInfo(W);
11889
11890 // Use the first element on the right as pivot since we will make less-than
11891 // comparisons against it.
11892 CaseClusterIt PivotCluster = FirstRight;
11893 assert(PivotCluster > W.FirstCluster);
11894 assert(PivotCluster <= W.LastCluster);
11895
11896 CaseClusterIt FirstLeft = W.FirstCluster;
11897 CaseClusterIt LastRight = W.LastCluster;
11898
11899 const ConstantInt *Pivot = PivotCluster->Low;
11900
11901 // New blocks will be inserted immediately after the current one.
11903 ++BBI;
11904
11905 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
11906 // we can branch to its destination directly if it's squeezed exactly in
11907 // between the known lower bound and Pivot - 1.
11908 MachineBasicBlock *LeftMBB;
11909 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
11910 FirstLeft->Low == W.GE &&
11911 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
11912 LeftMBB = FirstLeft->MBB;
11913 } else {
11914 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11915 FuncInfo.MF->insert(BBI, LeftMBB);
11916 WorkList.push_back(
11917 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
11918 // Put Cond in a virtual register to make it available from the new blocks.
11920 }
11921
11922 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
11923 // single cluster, RHS.Low == Pivot, and we can branch to its destination
11924 // directly if RHS.High equals the current upper bound.
11925 MachineBasicBlock *RightMBB;
11926 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
11927 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
11928 RightMBB = FirstRight->MBB;
11929 } else {
11930 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11931 FuncInfo.MF->insert(BBI, RightMBB);
11932 WorkList.push_back(
11933 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
11934 // Put Cond in a virtual register to make it available from the new blocks.
11936 }
11937
11938 // Create the CaseBlock record that will be used to lower the branch.
11939 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
11940 getCurSDLoc(), LeftProb, RightProb);
11941
11942 if (W.MBB == SwitchMBB)
11943 visitSwitchCase(CB, SwitchMBB);
11944 else
11945 SL->SwitchCases.push_back(CB);
11946}
11947
11948// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11949// from the swith statement.
11951 BranchProbability PeeledCaseProb) {
11952 if (PeeledCaseProb == BranchProbability::getOne())
11954 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
11955
11956 uint32_t Numerator = CaseProb.getNumerator();
11957 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
11958 return BranchProbability(Numerator, std::max(Numerator, Denominator));
11959}
11960
11961// Try to peel the top probability case if it exceeds the threshold.
11962// Return current MachineBasicBlock for the switch statement if the peeling
11963// does not occur.
11964// If the peeling is performed, return the newly created MachineBasicBlock
11965// for the peeled switch statement. Also update Clusters to remove the peeled
11966// case. PeeledCaseProb is the BranchProbability for the peeled case.
11967MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
11968 const SwitchInst &SI, CaseClusterVector &Clusters,
11969 BranchProbability &PeeledCaseProb) {
11970 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11971 // Don't perform if there is only one cluster or optimizing for size.
11972 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
11974 SwitchMBB->getParent()->getFunction().hasMinSize())
11975 return SwitchMBB;
11976
11978 unsigned PeeledCaseIndex = 0;
11979 bool SwitchPeeled = false;
11980 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
11981 CaseCluster &CC = Clusters[Index];
11982 if (CC.Prob < TopCaseProb)
11983 continue;
11984 TopCaseProb = CC.Prob;
11985 PeeledCaseIndex = Index;
11986 SwitchPeeled = true;
11987 }
11988 if (!SwitchPeeled)
11989 return SwitchMBB;
11990
11991 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
11992 << TopCaseProb << "\n");
11993
11994 // Record the MBB for the peeled switch statement.
11995 MachineFunction::iterator BBI(SwitchMBB);
11996 ++BBI;
11997 MachineBasicBlock *PeeledSwitchMBB =
11999 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
12000
12001 ExportFromCurrentBlock(SI.getCondition());
12002 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12003 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
12004 nullptr, nullptr, TopCaseProb.getCompl()};
12005 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
12006
12007 Clusters.erase(PeeledCaseIt);
12008 for (CaseCluster &CC : Clusters) {
12009 LLVM_DEBUG(
12010 dbgs() << "Scale the probablity for one cluster, before scaling: "
12011 << CC.Prob << "\n");
12012 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
12013 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12014 }
12015 PeeledCaseProb = TopCaseProb;
12016 return PeeledSwitchMBB;
12017}
12018
12019void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12020 // Extract cases from the switch.
12022 CaseClusterVector Clusters;
12023 Clusters.reserve(SI.getNumCases());
12024 for (auto I : SI.cases()) {
12025 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
12026 const ConstantInt *CaseVal = I.getCaseValue();
12027 BranchProbability Prob =
12028 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
12029 : BranchProbability(1, SI.getNumCases() + 1);
12030 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
12031 }
12032
12033 MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
12034
12035 // Cluster adjacent cases with the same destination. We do this at all
12036 // optimization levels because it's cheap to do and will make codegen faster
12037 // if there are many clusters.
12038 sortAndRangeify(Clusters);
12039
12040 // The branch probablity of the peeled case.
12042 MachineBasicBlock *PeeledSwitchMBB =
12043 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12044
12045 // If there is only the default destination, jump there directly.
12046 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12047 if (Clusters.empty()) {
12048 assert(PeeledSwitchMBB == SwitchMBB);
12049 SwitchMBB->addSuccessor(DefaultMBB);
12050 if (DefaultMBB != NextBlock(SwitchMBB)) {
12051 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
12052 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
12053 }
12054 return;
12055 }
12056
12057 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
12058 DAG.getBFI());
12059 SL->findBitTestClusters(Clusters, &SI);
12060
12061 LLVM_DEBUG({
12062 dbgs() << "Case clusters: ";
12063 for (const CaseCluster &C : Clusters) {
12064 if (C.Kind == CC_JumpTable)
12065 dbgs() << "JT:";
12066 if (C.Kind == CC_BitTests)
12067 dbgs() << "BT:";
12068
12069 C.Low->getValue().print(dbgs(), true);
12070 if (C.Low != C.High) {
12071 dbgs() << '-';
12072 C.High->getValue().print(dbgs(), true);
12073 }
12074 dbgs() << ' ';
12075 }
12076 dbgs() << '\n';
12077 });
12078
12079 assert(!Clusters.empty());
12080 SwitchWorkList WorkList;
12081 CaseClusterIt First = Clusters.begin();
12082 CaseClusterIt Last = Clusters.end() - 1;
12083 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
12084 // Scale the branchprobability for DefaultMBB if the peel occurs and
12085 // DefaultMBB is not replaced.
12086 if (PeeledCaseProb != BranchProbability::getZero() &&
12087 DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
12088 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
12089 WorkList.push_back(
12090 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
12091
12092 while (!WorkList.empty()) {
12093 SwitchWorkListItem W = WorkList.pop_back_val();
12094 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12095
12096 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12097 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12098 // For optimized builds, lower large range as a balanced binary tree.
12099 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
12100 continue;
12101 }
12102
12103 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
12104 }
12105}
12106
12107void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12109 auto DL = getCurSDLoc();
12110 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12111 setValue(&I, DAG.getStepVector(DL, ResultVT));
12112}
12113
12114void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12116 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12117
12118 SDLoc DL = getCurSDLoc();
12119 SDValue V = getValue(I.getOperand(0));
12120 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12121
12122 if (VT.isScalableVector()) {
12124 return;
12125 }
12126
12127 // Use VECTOR_SHUFFLE for the fixed-length vector
12128 // to maintain existing behavior.
12130 unsigned NumElts = VT.getVectorMinNumElements();
12131 for (unsigned i = 0; i != NumElts; ++i)
12132 Mask.push_back(NumElts - 1 - i);
12133
12134 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
12135}
12136
12137void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
12138 auto DL = getCurSDLoc();
12139 SDValue InVec = getValue(I.getOperand(0));
12140 EVT OutVT =
12142
12143 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12144
12145 // ISD Node needs the input vectors split into two equal parts
12146 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12148 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12149 DAG.getVectorIdxConstant(OutNumElts, DL));
12150
12151 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12152 // legalisation and combines.
12153 if (OutVT.isFixedLengthVector()) {
12154 SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12155 createStrideMask(0, 2, OutNumElts));
12156 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12157 createStrideMask(1, 2, OutNumElts));
12158 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
12159 setValue(&I, Res);
12160 return;
12161 }
12162
12164 DAG.getVTList(OutVT, OutVT), Lo, Hi);
12165 setValue(&I, Res);
12166}
12167
12168void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
12169 auto DL = getCurSDLoc();
12170 EVT InVT = getValue(I.getOperand(0)).getValueType();
12171 SDValue InVec0 = getValue(I.getOperand(0));
12172 SDValue InVec1 = getValue(I.getOperand(1));
12174 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12175
12176 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12177 // legalisation and combines.
12178 if (OutVT.isFixedLengthVector()) {
12179 unsigned NumElts = InVT.getVectorMinNumElements();
12180 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
12181 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
12182 createInterleaveMask(NumElts, 2)));
12183 return;
12184 }
12185
12187 DAG.getVTList(InVT, InVT), InVec0, InVec1);
12188 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
12189 Res.getValue(1));
12190 setValue(&I, Res);
12191}
12192
12193void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12194 SmallVector<EVT, 4> ValueVTs;
12196 ValueVTs);
12197 unsigned NumValues = ValueVTs.size();
12198 if (NumValues == 0) return;
12199
12200 SmallVector<SDValue, 4> Values(NumValues);
12201 SDValue Op = getValue(I.getOperand(0));
12202
12203 for (unsigned i = 0; i != NumValues; ++i)
12204 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
12205 SDValue(Op.getNode(), Op.getResNo() + i));
12206
12208 DAG.getVTList(ValueVTs), Values));
12209}
12210
12211void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12213 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12214
12215 SDLoc DL = getCurSDLoc();
12216 SDValue V1 = getValue(I.getOperand(0));
12217 SDValue V2 = getValue(I.getOperand(1));
12218 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
12219
12220 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
12221 if (VT.isScalableVector()) {
12222 MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
12223 setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
12224 DAG.getConstant(Imm, DL, IdxVT)));
12225 return;
12226 }
12227
12228 unsigned NumElts = VT.getVectorNumElements();
12229
12230 uint64_t Idx = (NumElts + Imm) % NumElts;
12231
12232 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12234 for (unsigned i = 0; i < NumElts; ++i)
12235 Mask.push_back(Idx + i);
12236 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
12237}
12238
12239// Consider the following MIR after SelectionDAG, which produces output in
12240// phyregs in the first case or virtregs in the second case.
12241//
12242// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12243// %5:gr32 = COPY $ebx
12244// %6:gr32 = COPY $edx
12245// %1:gr32 = COPY %6:gr32
12246// %0:gr32 = COPY %5:gr32
12247//
12248// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12249// %1:gr32 = COPY %6:gr32
12250// %0:gr32 = COPY %5:gr32
12251//
12252// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12253// Given %1, we'd like to return $edx in the first case and %6 in the second.
12254//
12255// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12256// to a single virtreg (such as %0). The remaining outputs monotonically
12257// increase in virtreg number from there. If a callbr has no outputs, then it
12258// should not have a corresponding callbr landingpad; in fact, the callbr
12259// landingpad would not even be able to refer to such a callbr.
12261 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
12262 // There is definitely at least one copy.
12263 assert(MI->getOpcode() == TargetOpcode::COPY &&
12264 "start of copy chain MUST be COPY");
12265 Reg = MI->getOperand(1).getReg();
12266 MI = MRI.def_begin(Reg)->getParent();
12267 // There may be an optional second copy.
12268 if (MI->getOpcode() == TargetOpcode::COPY) {
12269 assert(Reg.isVirtual() && "expected COPY of virtual register");
12270 Reg = MI->getOperand(1).getReg();
12271 assert(Reg.isPhysical() && "expected COPY of physical register");
12272 MI = MRI.def_begin(Reg)->getParent();
12273 }
12274 // The start of the chain must be an INLINEASM_BR.
12275 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12276 "end of copy chain MUST be INLINEASM_BR");
12277 return Reg;
12278}
12279
12280// We must do this walk rather than the simpler
12281// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
12282// otherwise we will end up with copies of virtregs only valid along direct
12283// edges.
12284void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
12285 SmallVector<EVT, 8> ResultVTs;
12286 SmallVector<SDValue, 8> ResultValues;
12287 const auto *CBR =
12288 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
12289
12293
12294 unsigned InitialDef = FuncInfo.ValueMap[CBR];
12295 SDValue Chain = DAG.getRoot();
12296
12297 // Re-parse the asm constraints string.
12298 TargetLowering::AsmOperandInfoVector TargetConstraints =
12299 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
12300 for (auto &T : TargetConstraints) {
12301 SDISelAsmOperandInfo OpInfo(T);
12302 if (OpInfo.Type != InlineAsm::isOutput)
12303 continue;
12304
12305 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
12306 // individual constraint.
12307 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
12308
12309 switch (OpInfo.ConstraintType) {
12312 // Fill in OpInfo.AssignedRegs.Regs.
12313 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
12314
12315 // getRegistersForValue may produce 1 to many registers based on whether
12316 // the OpInfo.ConstraintVT is legal on the target or not.
12317 for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) {
12318 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
12319 if (Register::isPhysicalRegister(OriginalDef))
12320 FuncInfo.MBB->addLiveIn(OriginalDef);
12321 // Update the assigned registers to use the original defs.
12322 OpInfo.AssignedRegs.Regs[i] = OriginalDef;
12323 }
12324
12325 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
12326 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
12327 ResultValues.push_back(V);
12328 ResultVTs.push_back(OpInfo.ConstraintVT);
12329 break;
12330 }
12332 SDValue Flag;
12333 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
12334 OpInfo, DAG);
12335 ++InitialDef;
12336 ResultValues.push_back(V);
12337 ResultVTs.push_back(OpInfo.ConstraintVT);
12338 break;
12339 }
12340 default:
12341 break;
12342 }
12343 }
12345 DAG.getVTList(ResultVTs), ResultValues);
12346 setValue(&I, V);
12347}
unsigned const MachineRegisterInfo * MRI
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition: FastISel.cpp:934
#define Check(C,...)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
This file provides utility analysis objects describing memory locations.
This file contains the declarations for metadata subclasses.
#define T1
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Module.h This file contains the declarations for the Module class.
uint64_t High
LLVMContext & Context
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< unsigned, TypeSize > > &Regs, const SDValue &N)
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, ISD::MemIndexType &IndexType, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static void findWasmUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1010
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
an instruction to allocate memory on the stack
Definition: Instructions.h:59
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:125
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition: Function.cpp:330
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:46
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
This class represents the atomic memcpy intrinsic i.e.
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
@ Add
*p = old + v
Definition: Instructions.h:764
@ FAdd
*p = old + v
Definition: Instructions.h:785
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:778
@ Or
*p = old | v
Definition: Instructions.h:772
@ Sub
*p = old - v
Definition: Instructions.h:766
@ And
*p = old & v
Definition: Instructions.h:768
@ Xor
*p = old ^ v
Definition: Instructions.h:774
@ FSub
*p = old - v
Definition: Instructions.h:788
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:800
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:776
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:782
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:796
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:780
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:792
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:804
@ Nand
*p = ~(old & v)
Definition: Instructions.h:770
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
bool getValueAsBool() const
Return the attribute's value as a boolean.
Definition: Attributes.cpp:335
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:347
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
Definition: BasicBlock.cpp:551
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:366
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:220
const Instruction & back() const
Definition: BasicBlock.h:454
This class represents a no-op cast from one type to another.
bool test(unsigned Idx) const
Definition: BitVector.h:461
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
BitVector & set()
Definition: BitVector.h:351
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
The address of a basic block.
Definition: Constants.h:888
Conditional or Unconditional Branch instruction.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static uint32_t getDenominator()
static BranchProbability getOne()
uint32_t getNumerator() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1455
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2357
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1761
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1623
bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2333
Value * getCalledOperand() const
Definition: InstrTypes.h:1696
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1648
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1629
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2241
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1561
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
unsigned arg_size() const
Definition: InstrTypes.h:1646
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1780
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:955
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition: Constants.h:582
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1016
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2140
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:267
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:204
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:856
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:144
This class represents a range of values.
Definition: ConstantRange.h:47
APInt getUnsignedMin() const
Return the smallest unsigned value contained in the ConstantRange.
bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
bool isEmptySet() const
Return true if this set contains no members.
bool isUpperWrapped() const
Return true if the exclusive upper bound wraps around the unsigned domain.
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition: Constants.h:506
This is an important base class in LLVM.
Definition: Constant.h:41
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Debug location.
Base class for variables.
std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
Records a position in IR for a source label (DILabel).
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
LocationType getType() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
bool isBigEndian() const
Definition: DataLayout.h:239
unsigned getAllocaAddrSpace() const
Definition: DataLayout.h:276
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
Definition: DataLayout.h:420
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:874
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
iterator_range< location_op_iterator > getValues() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
A debug info location.
Definition: DebugLoc.h:33
DILocation * getInlinedAt() const
Definition: DebugLoc.cpp:39
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:296
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition: TypeSize.h:302
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:307
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
bool allowReassoc() const
Flag queries.
Definition: FMF.h:65
An instruction for ordering other memory operations.
Definition: Instructions.h:460
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
Register CreateRegs(const Value *V)
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
BitVector DescribedArgs
Bitvector with a bit set if corresponding argument is described in ArgDbgValues.
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
SmallPtrSet< const DPValue *, 8 > PreprocessedDPVDeclares
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
Register InitializeRegForValue(const Value *V)
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineBasicBlock * MBB
MBB - The current block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
MachineRegisterInfo * RegInfo
Register CreateReg(MVT VT, bool isDivergent=false)
CreateReg - Allocate a single virtual register for the given type.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
Register getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:142
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
Type * getReturnType() const
Definition: DerivedTypes.h:124
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition: Function.h:782
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:695
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:230
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:677
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:330
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:262
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1874
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:338
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:235
size_t arg_size() const
Definition: Function.h:846
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition: Function.h:213
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:669
Garbage collection metadata for a single function.
Definition: GCMetadata.h:78
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Definition: GCMetadata.h:118
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:973
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:566
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
This instruction inserts a struct field of array element value into an aggregate value.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:453
const BasicBlock * getParent() const
Definition: Instruction.h:151
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1704
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Definition: DerivedTypes.h:51
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:184
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:321
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
Machine Value Type.
uint64_t getScalarSizeInBits() const
@ INVALID_SIMPLE_VALUE_TYPE
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void setIsEHCatchretTarget(bool V=true)
Indicates if this is a target block of a catchret.
void setIsCleanupFuncletEntry(bool V=true)
Indicates if this is the entry block of a cleanup funclet.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasPatchPoint(bool s=true)
void setHasStackMap(bool s=true)
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
Description of the location of a variable whose Address is valid and unchanging during function execu...
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
void setCallsUnwindInit(bool b)
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void setHasEHCatchret(bool V)
void setCallsEHReturn(bool b)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
auto getInStackSlotVariableDbgInfo()
Returns the collection of variables for which we have debug info and that have been assigned a stack ...
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
const MCContext & getContext() const
const Module * getModule() const
void setCurrentCallSite(unsigned Site)
Set the call site currently being processed.
unsigned getCurrentCallSite()
Get the call site currently being processed, if any.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
ArrayRef< std::pair< MCRegister, Register > > liveins() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool contains(const KeyT &Key) const
Definition: MapVector.h:163
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition: MapVector.h:118
Representation for a specific memory location.
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:176
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Definition: Module.cpp:331
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition: Operator.h:75
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1827
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed.
Definition: Operator.h:150
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
void CopyValueToVirtualRegister(const Value *V, unsigned Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, unsigned Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const BranchInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void init(GCFunctionInfo *gfi, AAResults *AA, AssumptionCache *AC, const TargetLibraryInfo *li)
DenseMap< const Constant *, unsigned > ConstantsOut
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineFunction * MF
virtual void emitFunctionEntryCode()
SwiftErrorValueTracking * SwiftError
std::unique_ptr< SelectionDAGBuilder > SDB
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
Help to insert SDNodeFlags automatically in transforming.
Definition: SelectionDAG.h:361
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:722
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:954
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:474
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
BlockFrequencyInfo * getBFI() const
Definition: SelectionDAG.h:488
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
bool shouldOptForSize() const
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
ProfileSummaryInfo * getPSI() const
Definition: SelectionDAG.h:487
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:727
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:480
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:862
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:773
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:708
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:799
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
Definition: SelectionDAG.h:484
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
void addPCSections(const SDNode *Node, MDNode *MD)
Set PCSections to be associated with Node.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:878
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:981
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
void clear()
Clear the memory usage of this object.
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
TargetIntrinsicInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *) const
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
std::vector< ArgListEntry > ArgListTy
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual const char * getClearCacheBuiltinName() const
Return the builtin name for the __builtin___clear_cache intrinsic Default is to invoke the clear cach...
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const
Return a TargetTransformInfo for a given function.
TargetOptions Options
CodeModel::Model getCodeModel() const
Returns the code model.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
unsigned getID() const
Return the register class ID number.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:361
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:750
bool isWasm() const
Tests whether the target is wasm (32- and 64-bit).
Definition: Triple.h:1005
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
Definition: Triple.h:895
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:255
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition: Type.h:216
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
Base class of all SIMD vector types.
Definition: DerivedTypes.h:403
Type * getElementType() const
Definition: DerivedTypes.h:436
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:86
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition: CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
Definition: CallingConv.h:245
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
Definition: CallingConv.h:163
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1126
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1122
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ CONVERGENCECTRL_ANCHOR
Definition: ISDOpcodes.h:1390
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1276
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:998
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1370
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:147
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1269
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1155
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1271
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1241
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1272
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1002
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1021
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:151
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1254
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1233
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1025
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1369
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FPTRUNC_ROUND
Definition: ISDOpcodes.h:480
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1267
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1268
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1199
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1274
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:135
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1108
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:885
@ CONVERGENCECTRL_GLUE
Definition: ISDOpcodes.h:1396
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ PREALLOCATED_SETUP
Definition: ISDOpcodes.h:1160
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1188
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:101
@ CONVERGENCECTRL_ENTRY
Definition: ISDOpcodes.h:1391
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:1047
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1221
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1275
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1270
@ PREALLOCATED_ARG
Definition: ISDOpcodes.h:1163
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:1052
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1056
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
Definition: ISDOpcodes.h:586
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:507
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1237
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1371
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1151
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1277
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1225
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1364
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:880
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:1117
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1016
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:993
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1265
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:118
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1211
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1329
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1273
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:114
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1215
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1041
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition: ISDOpcodes.h:591
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1372
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
Definition: ISDOpcodes.h:1174
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
Definition: ISDOpcodes.h:1097
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition: ISDOpcodes.h:129
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1279
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1263
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1264
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1182
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1208
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition: ISDOpcodes.h:106
@ CONVERGENCECTRL_LOOP
Definition: ISDOpcodes.h:1392
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1094
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:141
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1070
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:1113
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1278
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
Definition: ISDOpcodes.h:580
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1320
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1205
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1485
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:821
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
VScaleVal_match m_VScale()
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1522
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Reg
All possible values of the reg field in the ModR/M byte.
@ ReallyHidden
Definition: CommandLine.h:139
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:470
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:146
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebStrict
This corresponds to "fpexcept.strict".
Definition: FPEnv.h:41
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition: FPEnv.h:40
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
constexpr float log2ef
Definition: MathExtras.h:50
constexpr double e
Definition: MathExtras.h:31
constexpr float ln2f
Definition: MathExtras.h:48
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
@ Offset
Definition: DWP.cpp:456
@ Length
Definition: DWP.cpp:456
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1689
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
@ Done
Definition: Threading.h:61
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2082
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
gep_type_iterator gep_type_end(const User *GEP)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:342
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition: STLExtras.h:885
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition: STLExtras.h:322
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition: Analysis.cpp:199
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition: Local.cpp:2523
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2322
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
DWARFExpression::Operation Op
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition: Analysis.cpp:221
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2060
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:535
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2048
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:428
unsigned succ_size(const MachineBasicBlock *BB)
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:33
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition: bit.h:327
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
#define NC
Definition: regutils.h:42
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:249
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
uint64_t getScalarStoreSize() const
Definition: ValueTypes.h:387
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:349
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:366
bool isScalableVT() const
Return true if the type is a scalable type.
Definition: ValueTypes.h:183
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:156
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:101
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition: InlineAsm.h:126
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:244
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< unsigned, 4 > Regs
This list holds the registers assigned to the values.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< std::pair< unsigned, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
bool hasAllowReassociation() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
A cluster of case labels.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)