LLVM 20.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
84#include "llvm/IR/Metadata.h"
85#include "llvm/IR/Module.h"
86#include "llvm/IR/Operator.h"
88#include "llvm/IR/Statepoint.h"
89#include "llvm/IR/Type.h"
90#include "llvm/IR/User.h"
91#include "llvm/IR/Value.h"
92#include "llvm/MC/MCContext.h"
97#include "llvm/Support/Debug.h"
106#include <cstddef>
107#include <deque>
108#include <iterator>
109#include <limits>
110#include <optional>
111#include <tuple>
112
113using namespace llvm;
114using namespace PatternMatch;
115using namespace SwitchCG;
116
117#define DEBUG_TYPE "isel"
118
119/// LimitFloatPrecision - Generate low-precision inline sequences for
120/// some float libcalls (6, 8 or 12 bits).
121static unsigned LimitFloatPrecision;
122
123static cl::opt<bool>
124 InsertAssertAlign("insert-assert-align", cl::init(true),
125 cl::desc("Insert the experimental `assertalign` node."),
127
129 LimitFPPrecision("limit-float-precision",
130 cl::desc("Generate low-precision inline sequences "
131 "for some float libcalls"),
133 cl::init(0));
134
136 "switch-peel-threshold", cl::Hidden, cl::init(66),
137 cl::desc("Set the case probability threshold for peeling the case from a "
138 "switch statement. A value greater than 100 will void this "
139 "optimization"));
140
141// Limit the width of DAG chains. This is important in general to prevent
142// DAG-based analysis from blowing up. For example, alias analysis and
143// load clustering may not complete in reasonable time. It is difficult to
144// recognize and avoid this situation within each individual analysis, and
145// future analyses are likely to have the same behavior. Limiting DAG width is
146// the safe approach and will be especially important with global DAGs.
147//
148// MaxParallelChains default is arbitrarily high to avoid affecting
149// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
150// sequence over this should have been converted to llvm.memcpy by the
151// frontend. It is easy to induce this behavior with .ll code such as:
152// %buffer = alloca [4096 x i8]
153// %data = load [4096 x i8]* %argPtr
154// store [4096 x i8] %data, [4096 x i8]* %buffer
155static const unsigned MaxParallelChains = 64;
156
158 const SDValue *Parts, unsigned NumParts,
159 MVT PartVT, EVT ValueVT, const Value *V,
160 SDValue InChain,
161 std::optional<CallingConv::ID> CC);
162
163/// getCopyFromParts - Create a value that contains the specified legal parts
164/// combined into the value they represent. If the parts combine to a type
165/// larger than ValueVT then AssertOp can be used to specify whether the extra
166/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
167/// (ISD::AssertSext).
168static SDValue
169getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
170 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
171 SDValue InChain,
172 std::optional<CallingConv::ID> CC = std::nullopt,
173 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
174 // Let the target assemble the parts if it wants to
175 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
176 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
177 PartVT, ValueVT, CC))
178 return Val;
179
180 if (ValueVT.isVector())
181 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
182 InChain, CC);
183
184 assert(NumParts > 0 && "No parts to assemble!");
185 SDValue Val = Parts[0];
186
187 if (NumParts > 1) {
188 // Assemble the value from multiple parts.
189 if (ValueVT.isInteger()) {
190 unsigned PartBits = PartVT.getSizeInBits();
191 unsigned ValueBits = ValueVT.getSizeInBits();
192
193 // Assemble the power of 2 part.
194 unsigned RoundParts = llvm::bit_floor(NumParts);
195 unsigned RoundBits = PartBits * RoundParts;
196 EVT RoundVT = RoundBits == ValueBits ?
197 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
198 SDValue Lo, Hi;
199
200 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
201
202 if (RoundParts > 2) {
203 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
204 InChain);
205 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
206 PartVT, HalfVT, V, InChain);
207 } else {
208 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
209 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
210 }
211
212 if (DAG.getDataLayout().isBigEndian())
213 std::swap(Lo, Hi);
214
215 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
216
217 if (RoundParts < NumParts) {
218 // Assemble the trailing non-power-of-2 part.
219 unsigned OddParts = NumParts - RoundParts;
220 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
221 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
222 OddVT, V, InChain, CC);
223
224 // Combine the round and odd parts.
225 Lo = Val;
226 if (DAG.getDataLayout().isBigEndian())
227 std::swap(Lo, Hi);
228 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
229 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
230 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
231 DAG.getConstant(Lo.getValueSizeInBits(), DL,
233 TotalVT, DAG.getDataLayout())));
234 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
235 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
236 }
237 } else if (PartVT.isFloatingPoint()) {
238 // FP split into multiple FP parts (for ppcf128)
239 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
240 "Unexpected split");
241 SDValue Lo, Hi;
242 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
243 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
244 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
245 std::swap(Lo, Hi);
246 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
247 } else {
248 // FP split into integer parts (soft fp)
249 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
250 !PartVT.isVector() && "Unexpected split");
251 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
252 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
253 InChain, CC);
254 }
255 }
256
257 // There is now one part, held in Val. Correct it to match ValueVT.
258 // PartEVT is the type of the register class that holds the value.
259 // ValueVT is the type of the inline asm operation.
260 EVT PartEVT = Val.getValueType();
261
262 if (PartEVT == ValueVT)
263 return Val;
264
265 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
266 ValueVT.bitsLT(PartEVT)) {
267 // For an FP value in an integer part, we need to truncate to the right
268 // width first.
269 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
270 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
271 }
272
273 // Handle types that have the same size.
274 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
275 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
276
277 // Handle types with different sizes.
278 if (PartEVT.isInteger() && ValueVT.isInteger()) {
279 if (ValueVT.bitsLT(PartEVT)) {
280 // For a truncate, see if we have any information to
281 // indicate whether the truncated bits will always be
282 // zero or sign-extension.
283 if (AssertOp)
284 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
285 DAG.getValueType(ValueVT));
286 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
287 }
288 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
289 }
290
291 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
292 // FP_ROUND's are always exact here.
293 if (ValueVT.bitsLT(Val.getValueType())) {
294
295 SDValue NoChange =
297
299 llvm::Attribute::StrictFP)) {
300 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
301 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
302 NoChange);
303 }
304
305 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
306 }
307
308 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
309 }
310
311 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
312 // then truncating.
313 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
314 ValueVT.bitsLT(PartEVT)) {
315 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
316 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
317 }
318
319 report_fatal_error("Unknown mismatch in getCopyFromParts!");
320}
321
323 const Twine &ErrMsg) {
324 const Instruction *I = dyn_cast_or_null<Instruction>(V);
325 if (!V)
326 return Ctx.emitError(ErrMsg);
327
328 const char *AsmError = ", possible invalid constraint for vector type";
329 if (const CallInst *CI = dyn_cast<CallInst>(I))
330 if (CI->isInlineAsm())
331 return Ctx.emitError(I, ErrMsg + AsmError);
332
333 return Ctx.emitError(I, ErrMsg);
334}
335
336/// getCopyFromPartsVector - Create a value that contains the specified legal
337/// parts combined into the value they represent. If the parts combine to a
338/// type larger than ValueVT then AssertOp can be used to specify whether the
339/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
340/// ValueVT (ISD::AssertSext).
342 const SDValue *Parts, unsigned NumParts,
343 MVT PartVT, EVT ValueVT, const Value *V,
344 SDValue InChain,
345 std::optional<CallingConv::ID> CallConv) {
346 assert(ValueVT.isVector() && "Not a vector value");
347 assert(NumParts > 0 && "No parts to assemble!");
348 const bool IsABIRegCopy = CallConv.has_value();
349
350 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
351 SDValue Val = Parts[0];
352
353 // Handle a multi-element vector.
354 if (NumParts > 1) {
355 EVT IntermediateVT;
356 MVT RegisterVT;
357 unsigned NumIntermediates;
358 unsigned NumRegs;
359
360 if (IsABIRegCopy) {
362 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
363 NumIntermediates, RegisterVT);
364 } else {
365 NumRegs =
366 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
367 NumIntermediates, RegisterVT);
368 }
369
370 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
371 NumParts = NumRegs; // Silence a compiler warning.
372 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
373 assert(RegisterVT.getSizeInBits() ==
374 Parts[0].getSimpleValueType().getSizeInBits() &&
375 "Part type sizes don't match!");
376
377 // Assemble the parts into intermediate operands.
378 SmallVector<SDValue, 8> Ops(NumIntermediates);
379 if (NumIntermediates == NumParts) {
380 // If the register was not expanded, truncate or copy the value,
381 // as appropriate.
382 for (unsigned i = 0; i != NumParts; ++i)
383 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
384 V, InChain, CallConv);
385 } else if (NumParts > 0) {
386 // If the intermediate type was expanded, build the intermediate
387 // operands from the parts.
388 assert(NumParts % NumIntermediates == 0 &&
389 "Must expand into a divisible number of parts!");
390 unsigned Factor = NumParts / NumIntermediates;
391 for (unsigned i = 0; i != NumIntermediates; ++i)
392 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
393 IntermediateVT, V, InChain, CallConv);
394 }
395
396 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
397 // intermediate operands.
398 EVT BuiltVectorTy =
399 IntermediateVT.isVector()
401 *DAG.getContext(), IntermediateVT.getScalarType(),
402 IntermediateVT.getVectorElementCount() * NumParts)
404 IntermediateVT.getScalarType(),
405 NumIntermediates);
406 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
408 DL, BuiltVectorTy, Ops);
409 }
410
411 // There is now one part, held in Val. Correct it to match ValueVT.
412 EVT PartEVT = Val.getValueType();
413
414 if (PartEVT == ValueVT)
415 return Val;
416
417 if (PartEVT.isVector()) {
418 // Vector/Vector bitcast.
419 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
420 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
421
422 // If the parts vector has more elements than the value vector, then we
423 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
424 // Extract the elements we want.
425 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
428 (PartEVT.getVectorElementCount().isScalable() ==
429 ValueVT.getVectorElementCount().isScalable()) &&
430 "Cannot narrow, it would be a lossy transformation");
431 PartEVT =
433 ValueVT.getVectorElementCount());
434 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
435 DAG.getVectorIdxConstant(0, DL));
436 if (PartEVT == ValueVT)
437 return Val;
438 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
439 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
440
441 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
442 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
443 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
444 }
445
446 // Promoted vector extract
447 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
448 }
449
450 // Trivial bitcast if the types are the same size and the destination
451 // vector type is legal.
452 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
453 TLI.isTypeLegal(ValueVT))
454 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
455
456 if (ValueVT.getVectorNumElements() != 1) {
457 // Certain ABIs require that vectors are passed as integers. For vectors
458 // are the same size, this is an obvious bitcast.
459 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
460 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
461 } else if (ValueVT.bitsLT(PartEVT)) {
462 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
463 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
464 // Drop the extra bits.
465 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
466 return DAG.getBitcast(ValueVT, Val);
467 }
468
470 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
471 return DAG.getUNDEF(ValueVT);
472 }
473
474 // Handle cases such as i8 -> <1 x i1>
475 EVT ValueSVT = ValueVT.getVectorElementType();
476 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
477 unsigned ValueSize = ValueSVT.getSizeInBits();
478 if (ValueSize == PartEVT.getSizeInBits()) {
479 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
480 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
481 // It's possible a scalar floating point type gets softened to integer and
482 // then promoted to a larger integer. If PartEVT is the larger integer
483 // we need to truncate it and then bitcast to the FP type.
484 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
485 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
486 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
487 Val = DAG.getBitcast(ValueSVT, Val);
488 } else {
489 Val = ValueVT.isFloatingPoint()
490 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
491 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
492 }
493 }
494
495 return DAG.getBuildVector(ValueVT, DL, Val);
496}
497
498static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
499 SDValue Val, SDValue *Parts, unsigned NumParts,
500 MVT PartVT, const Value *V,
501 std::optional<CallingConv::ID> CallConv);
502
503/// getCopyToParts - Create a series of nodes that contain the specified value
504/// split into legal parts. If the parts contain more bits than Val, then, for
505/// integers, ExtendKind can be used to specify how to generate the extra bits.
506static void
508 unsigned NumParts, MVT PartVT, const Value *V,
509 std::optional<CallingConv::ID> CallConv = std::nullopt,
510 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
511 // Let the target split the parts if it wants to
512 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
513 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
514 CallConv))
515 return;
516 EVT ValueVT = Val.getValueType();
517
518 // Handle the vector case separately.
519 if (ValueVT.isVector())
520 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
521 CallConv);
522
523 unsigned OrigNumParts = NumParts;
525 "Copying to an illegal type!");
526
527 if (NumParts == 0)
528 return;
529
530 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
531 EVT PartEVT = PartVT;
532 if (PartEVT == ValueVT) {
533 assert(NumParts == 1 && "No-op copy with multiple parts!");
534 Parts[0] = Val;
535 return;
536 }
537
538 unsigned PartBits = PartVT.getSizeInBits();
539 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
540 // If the parts cover more bits than the value has, promote the value.
541 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
542 assert(NumParts == 1 && "Do not know what to promote to!");
543 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
544 } else {
545 if (ValueVT.isFloatingPoint()) {
546 // FP values need to be bitcast, then extended if they are being put
547 // into a larger container.
548 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
549 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
550 }
551 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
552 ValueVT.isInteger() &&
553 "Unknown mismatch!");
554 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
555 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
556 if (PartVT == MVT::x86mmx)
557 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
558 }
559 } else if (PartBits == ValueVT.getSizeInBits()) {
560 // Different types of the same size.
561 assert(NumParts == 1 && PartEVT != ValueVT);
562 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
563 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
564 // If the parts cover less bits than value has, truncate the value.
565 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
566 ValueVT.isInteger() &&
567 "Unknown mismatch!");
568 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
569 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
570 if (PartVT == MVT::x86mmx)
571 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
572 }
573
574 // The value may have changed - recompute ValueVT.
575 ValueVT = Val.getValueType();
576 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
577 "Failed to tile the value with PartVT!");
578
579 if (NumParts == 1) {
580 if (PartEVT != ValueVT) {
582 "scalar-to-vector conversion failed");
583 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
584 }
585
586 Parts[0] = Val;
587 return;
588 }
589
590 // Expand the value into multiple parts.
591 if (NumParts & (NumParts - 1)) {
592 // The number of parts is not a power of 2. Split off and copy the tail.
593 assert(PartVT.isInteger() && ValueVT.isInteger() &&
594 "Do not know what to expand to!");
595 unsigned RoundParts = llvm::bit_floor(NumParts);
596 unsigned RoundBits = RoundParts * PartBits;
597 unsigned OddParts = NumParts - RoundParts;
598 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
599 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
600
601 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
602 CallConv);
603
604 if (DAG.getDataLayout().isBigEndian())
605 // The odd parts were reversed by getCopyToParts - unreverse them.
606 std::reverse(Parts + RoundParts, Parts + NumParts);
607
608 NumParts = RoundParts;
609 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
610 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
611 }
612
613 // The number of parts is a power of 2. Repeatedly bisect the value using
614 // EXTRACT_ELEMENT.
615 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
617 ValueVT.getSizeInBits()),
618 Val);
619
620 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
621 for (unsigned i = 0; i < NumParts; i += StepSize) {
622 unsigned ThisBits = StepSize * PartBits / 2;
623 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
624 SDValue &Part0 = Parts[i];
625 SDValue &Part1 = Parts[i+StepSize/2];
626
627 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
628 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
629 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
630 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
631
632 if (ThisBits == PartBits && ThisVT != PartVT) {
633 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
634 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
635 }
636 }
637 }
638
639 if (DAG.getDataLayout().isBigEndian())
640 std::reverse(Parts, Parts + OrigNumParts);
641}
642
644 const SDLoc &DL, EVT PartVT) {
645 if (!PartVT.isVector())
646 return SDValue();
647
648 EVT ValueVT = Val.getValueType();
649 EVT PartEVT = PartVT.getVectorElementType();
650 EVT ValueEVT = ValueVT.getVectorElementType();
651 ElementCount PartNumElts = PartVT.getVectorElementCount();
652 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
653
654 // We only support widening vectors with equivalent element types and
655 // fixed/scalable properties. If a target needs to widen a fixed-length type
656 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
657 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
658 PartNumElts.isScalable() != ValueNumElts.isScalable())
659 return SDValue();
660
661 // Have a try for bf16 because some targets share its ABI with fp16.
662 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
664 "Cannot widen to illegal type");
665 Val = DAG.getNode(ISD::BITCAST, DL,
666 ValueVT.changeVectorElementType(MVT::f16), Val);
667 } else if (PartEVT != ValueEVT) {
668 return SDValue();
669 }
670
671 // Widening a scalable vector to another scalable vector is done by inserting
672 // the vector into a larger undef one.
673 if (PartNumElts.isScalable())
674 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
675 Val, DAG.getVectorIdxConstant(0, DL));
676
677 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
678 // undef elements.
680 DAG.ExtractVectorElements(Val, Ops);
681 SDValue EltUndef = DAG.getUNDEF(PartEVT);
682 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
683
684 // FIXME: Use CONCAT for 2x -> 4x.
685 return DAG.getBuildVector(PartVT, DL, Ops);
686}
687
688/// getCopyToPartsVector - Create a series of nodes that contain the specified
689/// value split into legal parts.
690static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
691 SDValue Val, SDValue *Parts, unsigned NumParts,
692 MVT PartVT, const Value *V,
693 std::optional<CallingConv::ID> CallConv) {
694 EVT ValueVT = Val.getValueType();
695 assert(ValueVT.isVector() && "Not a vector");
696 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
697 const bool IsABIRegCopy = CallConv.has_value();
698
699 if (NumParts == 1) {
700 EVT PartEVT = PartVT;
701 if (PartEVT == ValueVT) {
702 // Nothing to do.
703 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
704 // Bitconvert vector->vector case.
705 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
706 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
707 Val = Widened;
708 } else if (PartVT.isVector() &&
710 ValueVT.getVectorElementType()) &&
711 PartEVT.getVectorElementCount() ==
712 ValueVT.getVectorElementCount()) {
713
714 // Promoted vector extract
715 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
716 } else if (PartEVT.isVector() &&
717 PartEVT.getVectorElementType() !=
718 ValueVT.getVectorElementType() &&
719 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
720 TargetLowering::TypeWidenVector) {
721 // Combination of widening and promotion.
722 EVT WidenVT =
724 PartVT.getVectorElementCount());
725 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
726 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
727 } else {
728 // Don't extract an integer from a float vector. This can happen if the
729 // FP type gets softened to integer and then promoted. The promotion
730 // prevents it from being picked up by the earlier bitcast case.
731 if (ValueVT.getVectorElementCount().isScalar() &&
732 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
733 // If we reach this condition and PartVT is FP, this means that
734 // ValueVT is also FP and both have a different size, otherwise we
735 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
736 // would be invalid since that would mean the smaller FP type has to
737 // be extended to the larger one.
738 if (PartVT.isFloatingPoint()) {
739 Val = DAG.getBitcast(ValueVT.getScalarType(), Val);
740 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
741 } else
742 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
743 DAG.getVectorIdxConstant(0, DL));
744 } else {
745 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
746 assert(PartVT.getFixedSizeInBits() > ValueSize &&
747 "lossy conversion of vector to scalar type");
748 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
749 Val = DAG.getBitcast(IntermediateType, Val);
750 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
751 }
752 }
753
754 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
755 Parts[0] = Val;
756 return;
757 }
758
759 // Handle a multi-element vector.
760 EVT IntermediateVT;
761 MVT RegisterVT;
762 unsigned NumIntermediates;
763 unsigned NumRegs;
764 if (IsABIRegCopy) {
766 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
767 RegisterVT);
768 } else {
769 NumRegs =
770 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
771 NumIntermediates, RegisterVT);
772 }
773
774 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
775 NumParts = NumRegs; // Silence a compiler warning.
776 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
777
778 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
779 "Mixing scalable and fixed vectors when copying in parts");
780
781 std::optional<ElementCount> DestEltCnt;
782
783 if (IntermediateVT.isVector())
784 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
785 else
786 DestEltCnt = ElementCount::getFixed(NumIntermediates);
787
788 EVT BuiltVectorTy = EVT::getVectorVT(
789 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
790
791 if (ValueVT == BuiltVectorTy) {
792 // Nothing to do.
793 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
794 // Bitconvert vector->vector case.
795 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
796 } else {
797 if (BuiltVectorTy.getVectorElementType().bitsGT(
798 ValueVT.getVectorElementType())) {
799 // Integer promotion.
800 ValueVT = EVT::getVectorVT(*DAG.getContext(),
801 BuiltVectorTy.getVectorElementType(),
802 ValueVT.getVectorElementCount());
803 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
804 }
805
806 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
807 Val = Widened;
808 }
809 }
810
811 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
812
813 // Split the vector into intermediate operands.
814 SmallVector<SDValue, 8> Ops(NumIntermediates);
815 for (unsigned i = 0; i != NumIntermediates; ++i) {
816 if (IntermediateVT.isVector()) {
817 // This does something sensible for scalable vectors - see the
818 // definition of EXTRACT_SUBVECTOR for further details.
819 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
820 Ops[i] =
821 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
822 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
823 } else {
824 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
825 DAG.getVectorIdxConstant(i, DL));
826 }
827 }
828
829 // Split the intermediate operands into legal parts.
830 if (NumParts == NumIntermediates) {
831 // If the register was not expanded, promote or copy the value,
832 // as appropriate.
833 for (unsigned i = 0; i != NumParts; ++i)
834 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
835 } else if (NumParts > 0) {
836 // If the intermediate type was expanded, split each the value into
837 // legal parts.
838 assert(NumIntermediates != 0 && "division by zero");
839 assert(NumParts % NumIntermediates == 0 &&
840 "Must expand into a divisible number of parts!");
841 unsigned Factor = NumParts / NumIntermediates;
842 for (unsigned i = 0; i != NumIntermediates; ++i)
843 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
844 CallConv);
845 }
846}
847
849 EVT valuevt, std::optional<CallingConv::ID> CC)
850 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
851 RegCount(1, regs.size()), CallConv(CC) {}
852
854 const DataLayout &DL, unsigned Reg, Type *Ty,
855 std::optional<CallingConv::ID> CC) {
856 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
857
858 CallConv = CC;
859
860 for (EVT ValueVT : ValueVTs) {
861 unsigned NumRegs =
863 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
864 : TLI.getNumRegisters(Context, ValueVT);
865 MVT RegisterVT =
867 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
868 : TLI.getRegisterType(Context, ValueVT);
869 for (unsigned i = 0; i != NumRegs; ++i)
870 Regs.push_back(Reg + i);
871 RegVTs.push_back(RegisterVT);
872 RegCount.push_back(NumRegs);
873 Reg += NumRegs;
874 }
875}
876
878 FunctionLoweringInfo &FuncInfo,
879 const SDLoc &dl, SDValue &Chain,
880 SDValue *Glue, const Value *V) const {
881 // A Value with type {} or [0 x %t] needs no registers.
882 if (ValueVTs.empty())
883 return SDValue();
884
885 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
886
887 // Assemble the legal parts into the final values.
888 SmallVector<SDValue, 4> Values(ValueVTs.size());
890 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
891 // Copy the legal parts from the registers.
892 EVT ValueVT = ValueVTs[Value];
893 unsigned NumRegs = RegCount[Value];
894 MVT RegisterVT = isABIMangled()
896 *DAG.getContext(), *CallConv, RegVTs[Value])
897 : RegVTs[Value];
898
899 Parts.resize(NumRegs);
900 for (unsigned i = 0; i != NumRegs; ++i) {
901 SDValue P;
902 if (!Glue) {
903 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
904 } else {
905 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
906 *Glue = P.getValue(2);
907 }
908
909 Chain = P.getValue(1);
910 Parts[i] = P;
911
912 // If the source register was virtual and if we know something about it,
913 // add an assert node.
914 if (!Register::isVirtualRegister(Regs[Part + i]) ||
915 !RegisterVT.isInteger())
916 continue;
917
919 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
920 if (!LOI)
921 continue;
922
923 unsigned RegSize = RegisterVT.getScalarSizeInBits();
924 unsigned NumSignBits = LOI->NumSignBits;
925 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
926
927 if (NumZeroBits == RegSize) {
928 // The current value is a zero.
929 // Explicitly express that as it would be easier for
930 // optimizations to kick in.
931 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
932 continue;
933 }
934
935 // FIXME: We capture more information than the dag can represent. For
936 // now, just use the tightest assertzext/assertsext possible.
937 bool isSExt;
938 EVT FromVT(MVT::Other);
939 if (NumZeroBits) {
940 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
941 isSExt = false;
942 } else if (NumSignBits > 1) {
943 FromVT =
944 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
945 isSExt = true;
946 } else {
947 continue;
948 }
949 // Add an assertion node.
950 assert(FromVT != MVT::Other);
951 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
952 RegisterVT, P, DAG.getValueType(FromVT));
953 }
954
955 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
956 RegisterVT, ValueVT, V, Chain, CallConv);
957 Part += NumRegs;
958 Parts.clear();
959 }
960
961 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
962}
963
965 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
966 const Value *V,
967 ISD::NodeType PreferredExtendType) const {
968 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
969 ISD::NodeType ExtendKind = PreferredExtendType;
970
971 // Get the list of the values's legal parts.
972 unsigned NumRegs = Regs.size();
973 SmallVector<SDValue, 8> Parts(NumRegs);
974 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
975 unsigned NumParts = RegCount[Value];
976
977 MVT RegisterVT = isABIMangled()
979 *DAG.getContext(), *CallConv, RegVTs[Value])
980 : RegVTs[Value];
981
982 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
983 ExtendKind = ISD::ZERO_EXTEND;
984
985 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
986 NumParts, RegisterVT, V, CallConv, ExtendKind);
987 Part += NumParts;
988 }
989
990 // Copy the parts into the registers.
991 SmallVector<SDValue, 8> Chains(NumRegs);
992 for (unsigned i = 0; i != NumRegs; ++i) {
993 SDValue Part;
994 if (!Glue) {
995 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
996 } else {
997 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
998 *Glue = Part.getValue(1);
999 }
1000
1001 Chains[i] = Part.getValue(0);
1002 }
1003
1004 if (NumRegs == 1 || Glue)
1005 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1006 // flagged to it. That is the CopyToReg nodes and the user are considered
1007 // a single scheduling unit. If we create a TokenFactor and return it as
1008 // chain, then the TokenFactor is both a predecessor (operand) of the
1009 // user as well as a successor (the TF operands are flagged to the user).
1010 // c1, f1 = CopyToReg
1011 // c2, f2 = CopyToReg
1012 // c3 = TokenFactor c1, c2
1013 // ...
1014 // = op c3, ..., f2
1015 Chain = Chains[NumRegs-1];
1016 else
1017 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1018}
1019
1021 unsigned MatchingIdx, const SDLoc &dl,
1022 SelectionDAG &DAG,
1023 std::vector<SDValue> &Ops) const {
1024 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1025
1026 InlineAsm::Flag Flag(Code, Regs.size());
1027 if (HasMatching)
1028 Flag.setMatchingOp(MatchingIdx);
1029 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
1030 // Put the register class of the virtual registers in the flag word. That
1031 // way, later passes can recompute register class constraints for inline
1032 // assembly as well as normal instructions.
1033 // Don't do this for tied operands that can use the regclass information
1034 // from the def.
1036 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1037 Flag.setRegClass(RC->getID());
1038 }
1039
1040 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1041 Ops.push_back(Res);
1042
1043 if (Code == InlineAsm::Kind::Clobber) {
1044 // Clobbers should always have a 1:1 mapping with registers, and may
1045 // reference registers that have illegal (e.g. vector) types. Hence, we
1046 // shouldn't try to apply any sort of splitting logic to them.
1047 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1048 "No 1:1 mapping from clobbers to regs?");
1050 (void)SP;
1051 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1052 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1053 assert(
1054 (Regs[I] != SP ||
1056 "If we clobbered the stack pointer, MFI should know about it.");
1057 }
1058 return;
1059 }
1060
1061 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1062 MVT RegisterVT = RegVTs[Value];
1063 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1064 RegisterVT);
1065 for (unsigned i = 0; i != NumRegs; ++i) {
1066 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1067 unsigned TheReg = Regs[Reg++];
1068 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1069 }
1070 }
1071}
1072
1076 unsigned I = 0;
1077 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1078 unsigned RegCount = std::get<0>(CountAndVT);
1079 MVT RegisterVT = std::get<1>(CountAndVT);
1080 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1081 for (unsigned E = I + RegCount; I != E; ++I)
1082 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1083 }
1084 return OutVec;
1085}
1086
1088 AssumptionCache *ac,
1089 const TargetLibraryInfo *li) {
1090 AA = aa;
1091 AC = ac;
1092 GFI = gfi;
1093 LibInfo = li;
1095 LPadToCallSiteMap.clear();
1097 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1099}
1100
1102 NodeMap.clear();
1103 UnusedArgNodeMap.clear();
1104 PendingLoads.clear();
1105 PendingExports.clear();
1106 PendingConstrainedFP.clear();
1107 PendingConstrainedFPStrict.clear();
1108 CurInst = nullptr;
1109 HasTailCall = false;
1110 SDNodeOrder = LowestSDNodeOrder;
1112}
1113
1115 DanglingDebugInfoMap.clear();
1116}
1117
1118// Update DAG root to include dependencies on Pending chains.
1119SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1120 SDValue Root = DAG.getRoot();
1121
1122 if (Pending.empty())
1123 return Root;
1124
1125 // Add current root to PendingChains, unless we already indirectly
1126 // depend on it.
1127 if (Root.getOpcode() != ISD::EntryToken) {
1128 unsigned i = 0, e = Pending.size();
1129 for (; i != e; ++i) {
1130 assert(Pending[i].getNode()->getNumOperands() > 1);
1131 if (Pending[i].getNode()->getOperand(0) == Root)
1132 break; // Don't add the root if we already indirectly depend on it.
1133 }
1134
1135 if (i == e)
1136 Pending.push_back(Root);
1137 }
1138
1139 if (Pending.size() == 1)
1140 Root = Pending[0];
1141 else
1142 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1143
1144 DAG.setRoot(Root);
1145 Pending.clear();
1146 return Root;
1147}
1148
1150 return updateRoot(PendingLoads);
1151}
1152
1154 // Chain up all pending constrained intrinsics together with all
1155 // pending loads, by simply appending them to PendingLoads and
1156 // then calling getMemoryRoot().
1157 PendingLoads.reserve(PendingLoads.size() +
1158 PendingConstrainedFP.size() +
1159 PendingConstrainedFPStrict.size());
1160 PendingLoads.append(PendingConstrainedFP.begin(),
1161 PendingConstrainedFP.end());
1162 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1163 PendingConstrainedFPStrict.end());
1164 PendingConstrainedFP.clear();
1165 PendingConstrainedFPStrict.clear();
1166 return getMemoryRoot();
1167}
1168
1170 // We need to emit pending fpexcept.strict constrained intrinsics,
1171 // so append them to the PendingExports list.
1172 PendingExports.append(PendingConstrainedFPStrict.begin(),
1173 PendingConstrainedFPStrict.end());
1174 PendingConstrainedFPStrict.clear();
1175 return updateRoot(PendingExports);
1176}
1177
1179 DILocalVariable *Variable,
1181 DebugLoc DL) {
1182 assert(Variable && "Missing variable");
1183
1184 // Check if address has undef value.
1185 if (!Address || isa<UndefValue>(Address) ||
1186 (Address->use_empty() && !isa<Argument>(Address))) {
1187 LLVM_DEBUG(
1188 dbgs()
1189 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1190 return;
1191 }
1192
1193 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1194
1195 SDValue &N = NodeMap[Address];
1196 if (!N.getNode() && isa<Argument>(Address))
1197 // Check unused arguments map.
1198 N = UnusedArgNodeMap[Address];
1199 SDDbgValue *SDV;
1200 if (N.getNode()) {
1201 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1202 Address = BCI->getOperand(0);
1203 // Parameters are handled specially.
1204 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1205 if (IsParameter && FINode) {
1206 // Byval parameter. We have a frame index at this point.
1207 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1208 /*IsIndirect*/ true, DL, SDNodeOrder);
1209 } else if (isa<Argument>(Address)) {
1210 // Address is an argument, so try to emit its dbg value using
1211 // virtual register info from the FuncInfo.ValueMap.
1212 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1213 FuncArgumentDbgValueKind::Declare, N);
1214 return;
1215 } else {
1216 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1217 true, DL, SDNodeOrder);
1218 }
1219 DAG.AddDbgValue(SDV, IsParameter);
1220 } else {
1221 // If Address is an argument then try to emit its dbg value using
1222 // virtual register info from the FuncInfo.ValueMap.
1223 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1224 FuncArgumentDbgValueKind::Declare, N)) {
1225 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1226 << " (could not emit func-arg dbg_value)\n");
1227 }
1228 }
1229 return;
1230}
1231
1233 // Add SDDbgValue nodes for any var locs here. Do so before updating
1234 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1235 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1236 // Add SDDbgValue nodes for any var locs here. Do so before updating
1237 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1238 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1239 It != End; ++It) {
1240 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1241 dropDanglingDebugInfo(Var, It->Expr);
1242 if (It->Values.isKillLocation(It->Expr)) {
1243 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1244 continue;
1245 }
1246 SmallVector<Value *> Values(It->Values.location_ops());
1247 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1248 It->Values.hasArgList())) {
1249 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1251 FnVarLocs->getDILocalVariable(It->VariableID),
1252 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1253 }
1254 }
1255 }
1256
1257 // We must skip DbgVariableRecords if they've already been processed above as
1258 // we have just emitted the debug values resulting from assignment tracking
1259 // analysis, making any existing DbgVariableRecords redundant (and probably
1260 // less correct). We still need to process DbgLabelRecords. This does sink
1261 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1262 // be important as it does so deterministcally and ordering between
1263 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1264 // printing).
1265 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1266 // Is there is any debug-info attached to this instruction, in the form of
1267 // DbgRecord non-instruction debug-info records.
1268 for (DbgRecord &DR : I.getDbgRecordRange()) {
1269 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1270 assert(DLR->getLabel() && "Missing label");
1271 SDDbgLabel *SDV =
1272 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1273 DAG.AddDbgLabel(SDV);
1274 continue;
1275 }
1276
1277 if (SkipDbgVariableRecords)
1278 continue;
1279 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
1280 DILocalVariable *Variable = DVR.getVariable();
1283
1285 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1286 continue;
1287 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1288 << "\n");
1290 DVR.getDebugLoc());
1291 continue;
1292 }
1293
1294 // A DbgVariableRecord with no locations is a kill location.
1296 if (Values.empty()) {
1298 SDNodeOrder);
1299 continue;
1300 }
1301
1302 // A DbgVariableRecord with an undef or absent location is also a kill
1303 // location.
1304 if (llvm::any_of(Values,
1305 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1307 SDNodeOrder);
1308 continue;
1309 }
1310
1311 bool IsVariadic = DVR.hasArgList();
1312 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1313 SDNodeOrder, IsVariadic)) {
1314 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1315 DVR.getDebugLoc(), SDNodeOrder);
1316 }
1317 }
1318}
1319
1321 visitDbgInfo(I);
1322
1323 // Set up outgoing PHI node register values before emitting the terminator.
1324 if (I.isTerminator()) {
1325 HandlePHINodesInSuccessorBlocks(I.getParent());
1326 }
1327
1328 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1329 if (!isa<DbgInfoIntrinsic>(I))
1330 ++SDNodeOrder;
1331
1332 CurInst = &I;
1333
1334 // Set inserted listener only if required.
1335 bool NodeInserted = false;
1336 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1337 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1338 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);
1339 if (PCSectionsMD || MMRA) {
1340 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1341 DAG, [&](SDNode *) { NodeInserted = true; });
1342 }
1343
1344 visit(I.getOpcode(), I);
1345
1346 if (!I.isTerminator() && !HasTailCall &&
1347 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1349
1350 // Handle metadata.
1351 if (PCSectionsMD || MMRA) {
1352 auto It = NodeMap.find(&I);
1353 if (It != NodeMap.end()) {
1354 if (PCSectionsMD)
1355 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1356 if (MMRA)
1357 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1358 } else if (NodeInserted) {
1359 // This should not happen; if it does, don't let it go unnoticed so we can
1360 // fix it. Relevant visit*() function is probably missing a setValue().
1361 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1362 << I.getModule()->getName() << "]\n";
1363 LLVM_DEBUG(I.dump());
1364 assert(false);
1365 }
1366 }
1367
1368 CurInst = nullptr;
1369}
1370
1371void SelectionDAGBuilder::visitPHI(const PHINode &) {
1372 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1373}
1374
1375void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1376 // Note: this doesn't use InstVisitor, because it has to work with
1377 // ConstantExpr's in addition to instructions.
1378 switch (Opcode) {
1379 default: llvm_unreachable("Unknown instruction type encountered!");
1380 // Build the switch statement using the Instruction.def file.
1381#define HANDLE_INST(NUM, OPCODE, CLASS) \
1382 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1383#include "llvm/IR/Instruction.def"
1384 }
1385}
1386
1388 DILocalVariable *Variable,
1389 DebugLoc DL, unsigned Order,
1392 // For variadic dbg_values we will now insert an undef.
1393 // FIXME: We can potentially recover these!
1395 for (const Value *V : Values) {
1396 auto *Undef = UndefValue::get(V->getType());
1398 }
1399 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1400 /*IsIndirect=*/false, DL, Order,
1401 /*IsVariadic=*/true);
1402 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1403 return true;
1404}
1405
1407 DILocalVariable *Var,
1408 DIExpression *Expr,
1409 bool IsVariadic, DebugLoc DL,
1410 unsigned Order) {
1411 if (IsVariadic) {
1412 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1413 return;
1414 }
1415 // TODO: Dangling debug info will eventually either be resolved or produce
1416 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1417 // between the original dbg.value location and its resolved DBG_VALUE,
1418 // which we should ideally fill with an extra Undef DBG_VALUE.
1419 assert(Values.size() == 1);
1420 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1421}
1422
1424 const DIExpression *Expr) {
1425 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1426 DIVariable *DanglingVariable = DDI.getVariable();
1427 DIExpression *DanglingExpr = DDI.getExpression();
1428 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1429 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1430 << printDDI(nullptr, DDI) << "\n");
1431 return true;
1432 }
1433 return false;
1434 };
1435
1436 for (auto &DDIMI : DanglingDebugInfoMap) {
1437 DanglingDebugInfoVector &DDIV = DDIMI.second;
1438
1439 // If debug info is to be dropped, run it through final checks to see
1440 // whether it can be salvaged.
1441 for (auto &DDI : DDIV)
1442 if (isMatchingDbgValue(DDI))
1443 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1444
1445 erase_if(DDIV, isMatchingDbgValue);
1446 }
1447}
1448
1449// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1450// generate the debug data structures now that we've seen its definition.
1452 SDValue Val) {
1453 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1454 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1455 return;
1456
1457 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1458 for (auto &DDI : DDIV) {
1459 DebugLoc DL = DDI.getDebugLoc();
1460 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1461 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1462 DILocalVariable *Variable = DDI.getVariable();
1463 DIExpression *Expr = DDI.getExpression();
1465 "Expected inlined-at fields to agree");
1466 SDDbgValue *SDV;
1467 if (Val.getNode()) {
1468 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1469 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1470 // we couldn't resolve it directly when examining the DbgValue intrinsic
1471 // in the first place we should not be more successful here). Unless we
1472 // have some test case that prove this to be correct we should avoid
1473 // calling EmitFuncArgumentDbgValue here.
1474 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1475 FuncArgumentDbgValueKind::Value, Val)) {
1476 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1477 << printDDI(V, DDI) << "\n");
1478 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1479 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1480 // inserted after the definition of Val when emitting the instructions
1481 // after ISel. An alternative could be to teach
1482 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1483 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1484 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1485 << ValSDNodeOrder << "\n");
1486 SDV = getDbgValue(Val, Variable, Expr, DL,
1487 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1488 DAG.AddDbgValue(SDV, false);
1489 } else
1490 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1491 << printDDI(V, DDI)
1492 << " in EmitFuncArgumentDbgValue\n");
1493 } else {
1494 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1495 << "\n");
1496 auto Undef = UndefValue::get(V->getType());
1497 auto SDV =
1498 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1499 DAG.AddDbgValue(SDV, false);
1500 }
1501 }
1502 DDIV.clear();
1503}
1504
1506 DanglingDebugInfo &DDI) {
1507 // TODO: For the variadic implementation, instead of only checking the fail
1508 // state of `handleDebugValue`, we need know specifically which values were
1509 // invalid, so that we attempt to salvage only those values when processing
1510 // a DIArgList.
1511 const Value *OrigV = V;
1512 DILocalVariable *Var = DDI.getVariable();
1513 DIExpression *Expr = DDI.getExpression();
1514 DebugLoc DL = DDI.getDebugLoc();
1515 unsigned SDOrder = DDI.getSDNodeOrder();
1516
1517 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1518 // that DW_OP_stack_value is desired.
1519 bool StackValue = true;
1520
1521 // Can this Value can be encoded without any further work?
1522 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1523 return;
1524
1525 // Attempt to salvage back through as many instructions as possible. Bail if
1526 // a non-instruction is seen, such as a constant expression or global
1527 // variable. FIXME: Further work could recover those too.
1528 while (isa<Instruction>(V)) {
1529 const Instruction &VAsInst = *cast<const Instruction>(V);
1530 // Temporary "0", awaiting real implementation.
1532 SmallVector<Value *, 4> AdditionalValues;
1533 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1534 Expr->getNumLocationOperands(), Ops,
1535 AdditionalValues);
1536 // If we cannot salvage any further, and haven't yet found a suitable debug
1537 // expression, bail out.
1538 if (!V)
1539 break;
1540
1541 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1542 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1543 // here for variadic dbg_values, remove that condition.
1544 if (!AdditionalValues.empty())
1545 break;
1546
1547 // New value and expr now represent this debuginfo.
1548 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1549
1550 // Some kind of simplification occurred: check whether the operand of the
1551 // salvaged debug expression can be encoded in this DAG.
1552 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1553 LLVM_DEBUG(
1554 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1555 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1556 return;
1557 }
1558 }
1559
1560 // This was the final opportunity to salvage this debug information, and it
1561 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1562 // any earlier variable location.
1563 assert(OrigV && "V shouldn't be null");
1564 auto *Undef = UndefValue::get(OrigV->getType());
1565 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1566 DAG.AddDbgValue(SDV, false);
1567 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1568 << printDDI(OrigV, DDI) << "\n");
1569}
1570
1572 DIExpression *Expr,
1573 DebugLoc DbgLoc,
1574 unsigned Order) {
1578 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1579 /*IsVariadic*/ false);
1580}
1581
1583 DILocalVariable *Var,
1584 DIExpression *Expr, DebugLoc DbgLoc,
1585 unsigned Order, bool IsVariadic) {
1586 if (Values.empty())
1587 return true;
1588
1589 // Filter EntryValue locations out early.
1590 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1591 return true;
1592
1593 SmallVector<SDDbgOperand> LocationOps;
1594 SmallVector<SDNode *> Dependencies;
1595 for (const Value *V : Values) {
1596 // Constant value.
1597 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1598 isa<ConstantPointerNull>(V)) {
1599 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1600 continue;
1601 }
1602
1603 // Look through IntToPtr constants.
1604 if (auto *CE = dyn_cast<ConstantExpr>(V))
1605 if (CE->getOpcode() == Instruction::IntToPtr) {
1606 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1607 continue;
1608 }
1609
1610 // If the Value is a frame index, we can create a FrameIndex debug value
1611 // without relying on the DAG at all.
1612 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1613 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1614 if (SI != FuncInfo.StaticAllocaMap.end()) {
1615 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1616 continue;
1617 }
1618 }
1619
1620 // Do not use getValue() in here; we don't want to generate code at
1621 // this point if it hasn't been done yet.
1622 SDValue N = NodeMap[V];
1623 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1624 N = UnusedArgNodeMap[V];
1625 if (N.getNode()) {
1626 // Only emit func arg dbg value for non-variadic dbg.values for now.
1627 if (!IsVariadic &&
1628 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1629 FuncArgumentDbgValueKind::Value, N))
1630 return true;
1631 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1632 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1633 // describe stack slot locations.
1634 //
1635 // Consider "int x = 0; int *px = &x;". There are two kinds of
1636 // interesting debug values here after optimization:
1637 //
1638 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1639 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1640 //
1641 // Both describe the direct values of their associated variables.
1642 Dependencies.push_back(N.getNode());
1643 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1644 continue;
1645 }
1646 LocationOps.emplace_back(
1647 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1648 continue;
1649 }
1650
1652 // Special rules apply for the first dbg.values of parameter variables in a
1653 // function. Identify them by the fact they reference Argument Values, that
1654 // they're parameters, and they are parameters of the current function. We
1655 // need to let them dangle until they get an SDNode.
1656 bool IsParamOfFunc =
1657 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1658 if (IsParamOfFunc)
1659 return false;
1660
1661 // The value is not used in this block yet (or it would have an SDNode).
1662 // We still want the value to appear for the user if possible -- if it has
1663 // an associated VReg, we can refer to that instead.
1664 auto VMI = FuncInfo.ValueMap.find(V);
1665 if (VMI != FuncInfo.ValueMap.end()) {
1666 unsigned Reg = VMI->second;
1667 // If this is a PHI node, it may be split up into several MI PHI nodes
1668 // (in FunctionLoweringInfo::set).
1669 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1670 V->getType(), std::nullopt);
1671 if (RFV.occupiesMultipleRegs()) {
1672 // FIXME: We could potentially support variadic dbg_values here.
1673 if (IsVariadic)
1674 return false;
1675 unsigned Offset = 0;
1676 unsigned BitsToDescribe = 0;
1677 if (auto VarSize = Var->getSizeInBits())
1678 BitsToDescribe = *VarSize;
1679 if (auto Fragment = Expr->getFragmentInfo())
1680 BitsToDescribe = Fragment->SizeInBits;
1681 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1682 // Bail out if all bits are described already.
1683 if (Offset >= BitsToDescribe)
1684 break;
1685 // TODO: handle scalable vectors.
1686 unsigned RegisterSize = RegAndSize.second;
1687 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1688 ? BitsToDescribe - Offset
1689 : RegisterSize;
1690 auto FragmentExpr = DIExpression::createFragmentExpression(
1691 Expr, Offset, FragmentSize);
1692 if (!FragmentExpr)
1693 continue;
1695 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);
1696 DAG.AddDbgValue(SDV, false);
1697 Offset += RegisterSize;
1698 }
1699 return true;
1700 }
1701 // We can use simple vreg locations for variadic dbg_values as well.
1702 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1703 continue;
1704 }
1705 // We failed to create a SDDbgOperand for V.
1706 return false;
1707 }
1708
1709 // We have created a SDDbgOperand for each Value in Values.
1710 assert(!LocationOps.empty());
1711 SDDbgValue *SDV =
1712 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1713 /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
1714 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1715 return true;
1716}
1717
1719 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1720 for (auto &Pair : DanglingDebugInfoMap)
1721 for (auto &DDI : Pair.second)
1722 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1724}
1725
1726/// getCopyFromRegs - If there was virtual register allocated for the value V
1727/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1730 SDValue Result;
1731
1732 if (It != FuncInfo.ValueMap.end()) {
1733 Register InReg = It->second;
1734
1736 DAG.getDataLayout(), InReg, Ty,
1737 std::nullopt); // This is not an ABI copy.
1738 SDValue Chain = DAG.getEntryNode();
1739 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1740 V);
1741 resolveDanglingDebugInfo(V, Result);
1742 }
1743
1744 return Result;
1745}
1746
1747/// getValue - Return an SDValue for the given Value.
1749 // If we already have an SDValue for this value, use it. It's important
1750 // to do this first, so that we don't create a CopyFromReg if we already
1751 // have a regular SDValue.
1752 SDValue &N = NodeMap[V];
1753 if (N.getNode()) return N;
1754
1755 // If there's a virtual register allocated and initialized for this
1756 // value, use it.
1757 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1758 return copyFromReg;
1759
1760 // Otherwise create a new SDValue and remember it.
1761 SDValue Val = getValueImpl(V);
1762 NodeMap[V] = Val;
1764 return Val;
1765}
1766
1767/// getNonRegisterValue - Return an SDValue for the given Value, but
1768/// don't look in FuncInfo.ValueMap for a virtual register.
1770 // If we already have an SDValue for this value, use it.
1771 SDValue &N = NodeMap[V];
1772 if (N.getNode()) {
1773 if (isIntOrFPConstant(N)) {
1774 // Remove the debug location from the node as the node is about to be used
1775 // in a location which may differ from the original debug location. This
1776 // is relevant to Constant and ConstantFP nodes because they can appear
1777 // as constant expressions inside PHI nodes.
1778 N->setDebugLoc(DebugLoc());
1779 }
1780 return N;
1781 }
1782
1783 // Otherwise create a new SDValue and remember it.
1784 SDValue Val = getValueImpl(V);
1785 NodeMap[V] = Val;
1787 return Val;
1788}
1789
1790/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1791/// Create an SDValue for the given value.
1794
1795 if (const Constant *C = dyn_cast<Constant>(V)) {
1796 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1797
1798 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1799 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1800
1801 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1802 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1803
1804 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
1806 getValue(CPA->getPointer()), getValue(CPA->getKey()),
1807 getValue(CPA->getAddrDiscriminator()),
1808 getValue(CPA->getDiscriminator()));
1809 }
1810
1811 if (isa<ConstantPointerNull>(C)) {
1812 unsigned AS = V->getType()->getPointerAddressSpace();
1813 return DAG.getConstant(0, getCurSDLoc(),
1814 TLI.getPointerTy(DAG.getDataLayout(), AS));
1815 }
1816
1817 if (match(C, m_VScale()))
1818 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1819
1820 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1821 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1822
1823 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1824 return DAG.getUNDEF(VT);
1825
1826 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1827 visit(CE->getOpcode(), *CE);
1828 SDValue N1 = NodeMap[V];
1829 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1830 return N1;
1831 }
1832
1833 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1835 for (const Use &U : C->operands()) {
1836 SDNode *Val = getValue(U).getNode();
1837 // If the operand is an empty aggregate, there are no values.
1838 if (!Val) continue;
1839 // Add each leaf value from the operand to the Constants list
1840 // to form a flattened list of all the values.
1841 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1842 Constants.push_back(SDValue(Val, i));
1843 }
1844
1846 }
1847
1848 if (const ConstantDataSequential *CDS =
1849 dyn_cast<ConstantDataSequential>(C)) {
1851 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1852 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1853 // Add each leaf value from the operand to the Constants list
1854 // to form a flattened list of all the values.
1855 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1856 Ops.push_back(SDValue(Val, i));
1857 }
1858
1859 if (isa<ArrayType>(CDS->getType()))
1860 return DAG.getMergeValues(Ops, getCurSDLoc());
1861 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1862 }
1863
1864 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1865 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1866 "Unknown struct or array constant!");
1867
1868 SmallVector<EVT, 4> ValueVTs;
1869 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1870 unsigned NumElts = ValueVTs.size();
1871 if (NumElts == 0)
1872 return SDValue(); // empty struct
1874 for (unsigned i = 0; i != NumElts; ++i) {
1875 EVT EltVT = ValueVTs[i];
1876 if (isa<UndefValue>(C))
1877 Constants[i] = DAG.getUNDEF(EltVT);
1878 else if (EltVT.isFloatingPoint())
1879 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1880 else
1881 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1882 }
1883
1885 }
1886
1887 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1888 return DAG.getBlockAddress(BA, VT);
1889
1890 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1891 return getValue(Equiv->getGlobalValue());
1892
1893 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1894 return getValue(NC->getGlobalValue());
1895
1896 if (VT == MVT::aarch64svcount) {
1897 assert(C->isNullValue() && "Can only zero this target type!");
1898 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1899 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1900 }
1901
1902 VectorType *VecTy = cast<VectorType>(V->getType());
1903
1904 // Now that we know the number and type of the elements, get that number of
1905 // elements into the Ops array based on what kind of constant it is.
1906 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1908 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1909 for (unsigned i = 0; i != NumElements; ++i)
1910 Ops.push_back(getValue(CV->getOperand(i)));
1911
1912 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1913 }
1914
1915 if (isa<ConstantAggregateZero>(C)) {
1916 EVT EltVT =
1918
1919 SDValue Op;
1920 if (EltVT.isFloatingPoint())
1921 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1922 else
1923 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1924
1925 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1926 }
1927
1928 llvm_unreachable("Unknown vector constant");
1929 }
1930
1931 // If this is a static alloca, generate it as the frameindex instead of
1932 // computation.
1933 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1935 FuncInfo.StaticAllocaMap.find(AI);
1936 if (SI != FuncInfo.StaticAllocaMap.end())
1937 return DAG.getFrameIndex(
1938 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1939 }
1940
1941 // If this is an instruction which fast-isel has deferred, select it now.
1942 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1944
1945 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1946 Inst->getType(), std::nullopt);
1947 SDValue Chain = DAG.getEntryNode();
1948 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1949 }
1950
1951 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1952 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1953
1954 if (const auto *BB = dyn_cast<BasicBlock>(V))
1955 return DAG.getBasicBlock(FuncInfo.getMBB(BB));
1956
1957 llvm_unreachable("Can't get register for value!");
1958}
1959
1960void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1962 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1963 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1964 bool IsSEH = isAsynchronousEHPersonality(Pers);
1965 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1966 if (!IsSEH)
1967 CatchPadMBB->setIsEHScopeEntry();
1968 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1969 if (IsMSVCCXX || IsCoreCLR)
1970 CatchPadMBB->setIsEHFuncletEntry();
1971}
1972
1973void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1974 // Update machine-CFG edge.
1975 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());
1976 FuncInfo.MBB->addSuccessor(TargetMBB);
1977 TargetMBB->setIsEHCatchretTarget(true);
1979
1981 bool IsSEH = isAsynchronousEHPersonality(Pers);
1982 if (IsSEH) {
1983 // If this is not a fall-through branch or optimizations are switched off,
1984 // emit the branch.
1985 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1987 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1988 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1989 return;
1990 }
1991
1992 // Figure out the funclet membership for the catchret's successor.
1993 // This will be used by the FuncletLayout pass to determine how to order the
1994 // BB's.
1995 // A 'catchret' returns to the outer scope's color.
1996 Value *ParentPad = I.getCatchSwitchParentPad();
1997 const BasicBlock *SuccessorColor;
1998 if (isa<ConstantTokenNone>(ParentPad))
1999 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2000 else
2001 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
2002 assert(SuccessorColor && "No parent funclet for catchret!");
2003 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(SuccessorColor);
2004 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2005
2006 // Create the terminator node.
2008 getControlRoot(), DAG.getBasicBlock(TargetMBB),
2009 DAG.getBasicBlock(SuccessorColorMBB));
2010 DAG.setRoot(Ret);
2011}
2012
2013void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2014 // Don't emit any special code for the cleanuppad instruction. It just marks
2015 // the start of an EH scope/funclet.
2018 if (Pers != EHPersonality::Wasm_CXX) {
2021 }
2022}
2023
2024// In wasm EH, even though a catchpad may not catch an exception if a tag does
2025// not match, it is OK to add only the first unwind destination catchpad to the
2026// successors, because there will be at least one invoke instruction within the
2027// catch scope that points to the next unwind destination, if one exists, so
2028// CFGSort cannot mess up with BB sorting order.
2029// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
2030// call within them, and catchpads only consisting of 'catch (...)' have a
2031// '__cxa_end_catch' call within them, both of which generate invokes in case
2032// the next unwind destination exists, i.e., the next unwind destination is not
2033// the caller.)
2034//
2035// Having at most one EH pad successor is also simpler and helps later
2036// transformations.
2037//
2038// For example,
2039// current:
2040// invoke void @foo to ... unwind label %catch.dispatch
2041// catch.dispatch:
2042// %0 = catchswitch within ... [label %catch.start] unwind label %next
2043// catch.start:
2044// ...
2045// ... in this BB or some other child BB dominated by this BB there will be an
2046// invoke that points to 'next' BB as an unwind destination
2047//
2048// next: ; We don't need to add this to 'current' BB's successor
2049// ...
2051 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2052 BranchProbability Prob,
2053 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2054 &UnwindDests) {
2055 while (EHPadBB) {
2056 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2057 if (isa<CleanupPadInst>(Pad)) {
2058 // Stop on cleanup pads.
2059 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2060 UnwindDests.back().first->setIsEHScopeEntry();
2061 break;
2062 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2063 // Add the catchpad handlers to the possible destinations. We don't
2064 // continue to the unwind destination of the catchswitch for wasm.
2065 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2066 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2067 UnwindDests.back().first->setIsEHScopeEntry();
2068 }
2069 break;
2070 } else {
2071 continue;
2072 }
2073 }
2074}
2075
2076/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2077/// many places it could ultimately go. In the IR, we have a single unwind
2078/// destination, but in the machine CFG, we enumerate all the possible blocks.
2079/// This function skips over imaginary basic blocks that hold catchswitch
2080/// instructions, and finds all the "real" machine
2081/// basic block destinations. As those destinations may not be successors of
2082/// EHPadBB, here we also calculate the edge probability to those destinations.
2083/// The passed-in Prob is the edge probability to EHPadBB.
2085 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2086 BranchProbability Prob,
2087 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2088 &UnwindDests) {
2089 EHPersonality Personality =
2091 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2092 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2093 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2094 bool IsSEH = isAsynchronousEHPersonality(Personality);
2095
2096 if (IsWasmCXX) {
2097 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
2098 assert(UnwindDests.size() <= 1 &&
2099 "There should be at most one unwind destination for wasm");
2100 return;
2101 }
2102
2103 while (EHPadBB) {
2104 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2105 BasicBlock *NewEHPadBB = nullptr;
2106 if (isa<LandingPadInst>(Pad)) {
2107 // Stop on landingpads. They are not funclets.
2108 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2109 break;
2110 } else if (isa<CleanupPadInst>(Pad)) {
2111 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2112 // personalities.
2113 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2114 UnwindDests.back().first->setIsEHScopeEntry();
2115 UnwindDests.back().first->setIsEHFuncletEntry();
2116 break;
2117 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2118 // Add the catchpad handlers to the possible destinations.
2119 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2120 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2121 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2122 if (IsMSVCCXX || IsCoreCLR)
2123 UnwindDests.back().first->setIsEHFuncletEntry();
2124 if (!IsSEH)
2125 UnwindDests.back().first->setIsEHScopeEntry();
2126 }
2127 NewEHPadBB = CatchSwitch->getUnwindDest();
2128 } else {
2129 continue;
2130 }
2131
2132 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2133 if (BPI && NewEHPadBB)
2134 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2135 EHPadBB = NewEHPadBB;
2136 }
2137}
2138
2139void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2140 // Update successor info.
2142 auto UnwindDest = I.getUnwindDest();
2144 BranchProbability UnwindDestProb =
2145 (BPI && UnwindDest)
2146 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2148 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2149 for (auto &UnwindDest : UnwindDests) {
2150 UnwindDest.first->setIsEHPad();
2151 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2152 }
2154
2155 // Create the terminator node.
2156 SDValue Ret =
2158 DAG.setRoot(Ret);
2159}
2160
2161void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2162 report_fatal_error("visitCatchSwitch not yet implemented!");
2163}
2164
2165void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2167 auto &DL = DAG.getDataLayout();
2168 SDValue Chain = getControlRoot();
2171
2172 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2173 // lower
2174 //
2175 // %val = call <ty> @llvm.experimental.deoptimize()
2176 // ret <ty> %val
2177 //
2178 // differently.
2179 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2181 return;
2182 }
2183
2184 if (!FuncInfo.CanLowerReturn) {
2185 unsigned DemoteReg = FuncInfo.DemoteRegister;
2186 const Function *F = I.getParent()->getParent();
2187
2188 // Emit a store of the return value through the virtual register.
2189 // Leave Outs empty so that LowerReturn won't try to load return
2190 // registers the usual way.
2191 SmallVector<EVT, 1> PtrValueVTs;
2192 ComputeValueVTs(TLI, DL,
2193 PointerType::get(F->getContext(),
2195 PtrValueVTs);
2196
2197 SDValue RetPtr =
2198 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2199 SDValue RetOp = getValue(I.getOperand(0));
2200
2201 SmallVector<EVT, 4> ValueVTs, MemVTs;
2203 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2204 &Offsets, 0);
2205 unsigned NumValues = ValueVTs.size();
2206
2207 SmallVector<SDValue, 4> Chains(NumValues);
2208 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2209 for (unsigned i = 0; i != NumValues; ++i) {
2210 // An aggregate return value cannot wrap around the address space, so
2211 // offsets to its parts don't wrap either.
2213 TypeSize::getFixed(Offsets[i]));
2214
2215 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2216 if (MemVTs[i] != ValueVTs[i])
2217 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2218 Chains[i] = DAG.getStore(
2219 Chain, getCurSDLoc(), Val,
2220 // FIXME: better loc info would be nice.
2222 commonAlignment(BaseAlign, Offsets[i]));
2223 }
2224
2226 MVT::Other, Chains);
2227 } else if (I.getNumOperands() != 0) {
2228 SmallVector<EVT, 4> ValueVTs;
2229 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2230 unsigned NumValues = ValueVTs.size();
2231 if (NumValues) {
2232 SDValue RetOp = getValue(I.getOperand(0));
2233
2234 const Function *F = I.getParent()->getParent();
2235
2236 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2237 I.getOperand(0)->getType(), F->getCallingConv(),
2238 /*IsVarArg*/ false, DL);
2239
2240 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2241 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2242 ExtendKind = ISD::SIGN_EXTEND;
2243 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2244 ExtendKind = ISD::ZERO_EXTEND;
2245
2246 LLVMContext &Context = F->getContext();
2247 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2248
2249 for (unsigned j = 0; j != NumValues; ++j) {
2250 EVT VT = ValueVTs[j];
2251
2252 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2253 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2254
2255 CallingConv::ID CC = F->getCallingConv();
2256
2257 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2258 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2259 SmallVector<SDValue, 4> Parts(NumParts);
2261 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2262 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2263
2264 // 'inreg' on function refers to return value
2266 if (RetInReg)
2267 Flags.setInReg();
2268
2269 if (I.getOperand(0)->getType()->isPointerTy()) {
2270 Flags.setPointer();
2271 Flags.setPointerAddrSpace(
2272 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2273 }
2274
2275 if (NeedsRegBlock) {
2276 Flags.setInConsecutiveRegs();
2277 if (j == NumValues - 1)
2278 Flags.setInConsecutiveRegsLast();
2279 }
2280
2281 // Propagate extension type if any
2282 if (ExtendKind == ISD::SIGN_EXTEND)
2283 Flags.setSExt();
2284 else if (ExtendKind == ISD::ZERO_EXTEND)
2285 Flags.setZExt();
2286
2287 for (unsigned i = 0; i < NumParts; ++i) {
2288 Outs.push_back(ISD::OutputArg(Flags,
2289 Parts[i].getValueType().getSimpleVT(),
2290 VT, /*isfixed=*/true, 0, 0));
2291 OutVals.push_back(Parts[i]);
2292 }
2293 }
2294 }
2295 }
2296
2297 // Push in swifterror virtual register as the last element of Outs. This makes
2298 // sure swifterror virtual register will be returned in the swifterror
2299 // physical register.
2300 const Function *F = I.getParent()->getParent();
2301 if (TLI.supportSwiftError() &&
2302 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2303 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2305 Flags.setSwiftError();
2307 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2308 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2309 // Create SDNode for the swifterror virtual register.
2310 OutVals.push_back(
2313 EVT(TLI.getPointerTy(DL))));
2314 }
2315
2316 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2317 CallingConv::ID CallConv =
2320 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2321
2322 // Verify that the target's LowerReturn behaved as expected.
2323 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2324 "LowerReturn didn't return a valid chain!");
2325
2326 // Update the DAG with the new chain value resulting from return lowering.
2327 DAG.setRoot(Chain);
2328}
2329
2330/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2331/// created for it, emit nodes to copy the value into the virtual
2332/// registers.
2334 // Skip empty types
2335 if (V->getType()->isEmptyTy())
2336 return;
2337
2339 if (VMI != FuncInfo.ValueMap.end()) {
2340 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2341 "Unused value assigned virtual registers!");
2342 CopyValueToVirtualRegister(V, VMI->second);
2343 }
2344}
2345
2346/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2347/// the current basic block, add it to ValueMap now so that we'll get a
2348/// CopyTo/FromReg.
2350 // No need to export constants.
2351 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2352
2353 // Already exported?
2354 if (FuncInfo.isExportedInst(V)) return;
2355
2358}
2359
2361 const BasicBlock *FromBB) {
2362 // The operands of the setcc have to be in this block. We don't know
2363 // how to export them from some other block.
2364 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2365 // Can export from current BB.
2366 if (VI->getParent() == FromBB)
2367 return true;
2368
2369 // Is already exported, noop.
2370 return FuncInfo.isExportedInst(V);
2371 }
2372
2373 // If this is an argument, we can export it if the BB is the entry block or
2374 // if it is already exported.
2375 if (isa<Argument>(V)) {
2376 if (FromBB->isEntryBlock())
2377 return true;
2378
2379 // Otherwise, can only export this if it is already exported.
2380 return FuncInfo.isExportedInst(V);
2381 }
2382
2383 // Otherwise, constants can always be exported.
2384 return true;
2385}
2386
2387/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2389SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2390 const MachineBasicBlock *Dst) const {
2392 const BasicBlock *SrcBB = Src->getBasicBlock();
2393 const BasicBlock *DstBB = Dst->getBasicBlock();
2394 if (!BPI) {
2395 // If BPI is not available, set the default probability as 1 / N, where N is
2396 // the number of successors.
2397 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2398 return BranchProbability(1, SuccSize);
2399 }
2400 return BPI->getEdgeProbability(SrcBB, DstBB);
2401}
2402
2403void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2404 MachineBasicBlock *Dst,
2405 BranchProbability Prob) {
2406 if (!FuncInfo.BPI)
2407 Src->addSuccessorWithoutProb(Dst);
2408 else {
2409 if (Prob.isUnknown())
2410 Prob = getEdgeProbability(Src, Dst);
2411 Src->addSuccessor(Dst, Prob);
2412 }
2413}
2414
2415static bool InBlock(const Value *V, const BasicBlock *BB) {
2416 if (const Instruction *I = dyn_cast<Instruction>(V))
2417 return I->getParent() == BB;
2418 return true;
2419}
2420
2421/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2422/// This function emits a branch and is used at the leaves of an OR or an
2423/// AND operator tree.
2424void
2427 MachineBasicBlock *FBB,
2428 MachineBasicBlock *CurBB,
2429 MachineBasicBlock *SwitchBB,
2430 BranchProbability TProb,
2431 BranchProbability FProb,
2432 bool InvertCond) {
2433 const BasicBlock *BB = CurBB->getBasicBlock();
2434
2435 // If the leaf of the tree is a comparison, merge the condition into
2436 // the caseblock.
2437 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2438 // The operands of the cmp have to be in this block. We don't know
2439 // how to export them from some other block. If this is the first block
2440 // of the sequence, no exporting is needed.
2441 if (CurBB == SwitchBB ||
2442 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2443 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2444 ISD::CondCode Condition;
2445 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2446 ICmpInst::Predicate Pred =
2447 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2448 Condition = getICmpCondCode(Pred);
2449 } else {
2450 const FCmpInst *FC = cast<FCmpInst>(Cond);
2451 FCmpInst::Predicate Pred =
2452 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2453 Condition = getFCmpCondCode(Pred);
2454 if (TM.Options.NoNaNsFPMath)
2455 Condition = getFCmpCodeWithoutNaN(Condition);
2456 }
2457
2458 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2459 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2460 SL->SwitchCases.push_back(CB);
2461 return;
2462 }
2463 }
2464
2465 // Create a CaseBlock record representing this branch.
2466 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2468 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2469 SL->SwitchCases.push_back(CB);
2470}
2471
2472// Collect dependencies on V recursively. This is used for the cost analysis in
2473// `shouldKeepJumpConditionsTogether`.
2477 unsigned Depth = 0) {
2478 // Return false if we have an incomplete count.
2480 return false;
2481
2482 auto *I = dyn_cast<Instruction>(V);
2483 if (I == nullptr)
2484 return true;
2485
2486 if (Necessary != nullptr) {
2487 // This instruction is necessary for the other side of the condition so
2488 // don't count it.
2489 if (Necessary->contains(I))
2490 return true;
2491 }
2492
2493 // Already added this dep.
2494 if (!Deps->try_emplace(I, false).second)
2495 return true;
2496
2497 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2498 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2499 Depth + 1))
2500 return false;
2501 return true;
2502}
2503
2505 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,
2506 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2508 if (I.getNumSuccessors() != 2)
2509 return false;
2510
2511 if (!I.isConditional())
2512 return false;
2513
2514 if (Params.BaseCost < 0)
2515 return false;
2516
2517 // Baseline cost.
2518 InstructionCost CostThresh = Params.BaseCost;
2519
2520 BranchProbabilityInfo *BPI = nullptr;
2521 if (Params.LikelyBias || Params.UnlikelyBias)
2522 BPI = FuncInfo.BPI;
2523 if (BPI != nullptr) {
2524 // See if we are either likely to get an early out or compute both lhs/rhs
2525 // of the condition.
2526 BasicBlock *IfFalse = I.getSuccessor(0);
2527 BasicBlock *IfTrue = I.getSuccessor(1);
2528
2529 std::optional<bool> Likely;
2530 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2531 Likely = true;
2532 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2533 Likely = false;
2534
2535 if (Likely) {
2536 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2537 // Its likely we will have to compute both lhs and rhs of condition
2538 CostThresh += Params.LikelyBias;
2539 else {
2540 if (Params.UnlikelyBias < 0)
2541 return false;
2542 // Its likely we will get an early out.
2543 CostThresh -= Params.UnlikelyBias;
2544 }
2545 }
2546 }
2547
2548 if (CostThresh <= 0)
2549 return false;
2550
2551 // Collect "all" instructions that lhs condition is dependent on.
2552 // Use map for stable iteration (to avoid non-determanism of iteration of
2553 // SmallPtrSet). The `bool` value is just a dummy.
2555 collectInstructionDeps(&LhsDeps, Lhs);
2556 // Collect "all" instructions that rhs condition is dependent on AND are
2557 // dependencies of lhs. This gives us an estimate on which instructions we
2558 // stand to save by splitting the condition.
2559 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2560 return false;
2561 // Add the compare instruction itself unless its a dependency on the LHS.
2562 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2563 if (!LhsDeps.contains(RhsI))
2564 RhsDeps.try_emplace(RhsI, false);
2565
2566 const auto &TLI = DAG.getTargetLoweringInfo();
2567 const auto &TTI =
2568 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
2569
2570 InstructionCost CostOfIncluding = 0;
2571 // See if this instruction will need to computed independently of whether RHS
2572 // is.
2573 Value *BrCond = I.getCondition();
2574 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2575 for (const auto *U : Ins->users()) {
2576 // If user is independent of RHS calculation we don't need to count it.
2577 if (auto *UIns = dyn_cast<Instruction>(U))
2578 if (UIns != BrCond && !RhsDeps.contains(UIns))
2579 return false;
2580 }
2581 return true;
2582 };
2583
2584 // Prune instructions from RHS Deps that are dependencies of unrelated
2585 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2586 // arbitrary and just meant to cap the how much time we spend in the pruning
2587 // loop. Its highly unlikely to come into affect.
2588 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2589 // Stop after a certain point. No incorrectness from including too many
2590 // instructions.
2591 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2592 const Instruction *ToDrop = nullptr;
2593 for (const auto &InsPair : RhsDeps) {
2594 if (!ShouldCountInsn(InsPair.first)) {
2595 ToDrop = InsPair.first;
2596 break;
2597 }
2598 }
2599 if (ToDrop == nullptr)
2600 break;
2601 RhsDeps.erase(ToDrop);
2602 }
2603
2604 for (const auto &InsPair : RhsDeps) {
2605 // Finally accumulate latency that we can only attribute to computing the
2606 // RHS condition. Use latency because we are essentially trying to calculate
2607 // the cost of the dependency chain.
2608 // Possible TODO: We could try to estimate ILP and make this more precise.
2609 CostOfIncluding +=
2611
2612 if (CostOfIncluding > CostThresh)
2613 return false;
2614 }
2615 return true;
2616}
2617
2620 MachineBasicBlock *FBB,
2621 MachineBasicBlock *CurBB,
2622 MachineBasicBlock *SwitchBB,
2624 BranchProbability TProb,
2625 BranchProbability FProb,
2626 bool InvertCond) {
2627 // Skip over not part of the tree and remember to invert op and operands at
2628 // next level.
2629 Value *NotCond;
2630 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2631 InBlock(NotCond, CurBB->getBasicBlock())) {
2632 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2633 !InvertCond);
2634 return;
2635 }
2636
2637 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2638 const Value *BOpOp0, *BOpOp1;
2639 // Compute the effective opcode for Cond, taking into account whether it needs
2640 // to be inverted, e.g.
2641 // and (not (or A, B)), C
2642 // gets lowered as
2643 // and (and (not A, not B), C)
2645 if (BOp) {
2646 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2647 ? Instruction::And
2648 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2649 ? Instruction::Or
2651 if (InvertCond) {
2652 if (BOpc == Instruction::And)
2653 BOpc = Instruction::Or;
2654 else if (BOpc == Instruction::Or)
2655 BOpc = Instruction::And;
2656 }
2657 }
2658
2659 // If this node is not part of the or/and tree, emit it as a branch.
2660 // Note that all nodes in the tree should have same opcode.
2661 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2662 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2663 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2664 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2665 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2666 TProb, FProb, InvertCond);
2667 return;
2668 }
2669
2670 // Create TmpBB after CurBB.
2671 MachineFunction::iterator BBI(CurBB);
2674 CurBB->getParent()->insert(++BBI, TmpBB);
2675
2676 if (Opc == Instruction::Or) {
2677 // Codegen X | Y as:
2678 // BB1:
2679 // jmp_if_X TBB
2680 // jmp TmpBB
2681 // TmpBB:
2682 // jmp_if_Y TBB
2683 // jmp FBB
2684 //
2685
2686 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2687 // The requirement is that
2688 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2689 // = TrueProb for original BB.
2690 // Assuming the original probabilities are A and B, one choice is to set
2691 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2692 // A/(1+B) and 2B/(1+B). This choice assumes that
2693 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2694 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2695 // TmpBB, but the math is more complicated.
2696
2697 auto NewTrueProb = TProb / 2;
2698 auto NewFalseProb = TProb / 2 + FProb;
2699 // Emit the LHS condition.
2700 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2701 NewFalseProb, InvertCond);
2702
2703 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2704 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2705 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2706 // Emit the RHS condition into TmpBB.
2707 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2708 Probs[1], InvertCond);
2709 } else {
2710 assert(Opc == Instruction::And && "Unknown merge op!");
2711 // Codegen X & Y as:
2712 // BB1:
2713 // jmp_if_X TmpBB
2714 // jmp FBB
2715 // TmpBB:
2716 // jmp_if_Y TBB
2717 // jmp FBB
2718 //
2719 // This requires creation of TmpBB after CurBB.
2720
2721 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2722 // The requirement is that
2723 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2724 // = FalseProb for original BB.
2725 // Assuming the original probabilities are A and B, one choice is to set
2726 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2727 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2728 // TrueProb for BB1 * FalseProb for TmpBB.
2729
2730 auto NewTrueProb = TProb + FProb / 2;
2731 auto NewFalseProb = FProb / 2;
2732 // Emit the LHS condition.
2733 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2734 NewFalseProb, InvertCond);
2735
2736 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2737 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2738 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2739 // Emit the RHS condition into TmpBB.
2740 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2741 Probs[1], InvertCond);
2742 }
2743}
2744
2745/// If the set of cases should be emitted as a series of branches, return true.
2746/// If we should emit this as a bunch of and/or'd together conditions, return
2747/// false.
2748bool
2749SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2750 if (Cases.size() != 2) return true;
2751
2752 // If this is two comparisons of the same values or'd or and'd together, they
2753 // will get folded into a single comparison, so don't emit two blocks.
2754 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2755 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2756 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2757 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2758 return false;
2759 }
2760
2761 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2762 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2763 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2764 Cases[0].CC == Cases[1].CC &&
2765 isa<Constant>(Cases[0].CmpRHS) &&
2766 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2767 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2768 return false;
2769 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2770 return false;
2771 }
2772
2773 return true;
2774}
2775
2776void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2778
2779 // Update machine-CFG edges.
2780 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2781
2782 if (I.isUnconditional()) {
2783 // Update machine-CFG edges.
2784 BrMBB->addSuccessor(Succ0MBB);
2785
2786 // If this is not a fall-through branch or optimizations are switched off,
2787 // emit the branch.
2788 if (Succ0MBB != NextBlock(BrMBB) ||
2790 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2791 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2792 setValue(&I, Br);
2793 DAG.setRoot(Br);
2794 }
2795
2796 return;
2797 }
2798
2799 // If this condition is one of the special cases we handle, do special stuff
2800 // now.
2801 const Value *CondVal = I.getCondition();
2802 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(I.getSuccessor(1));
2803
2804 // If this is a series of conditions that are or'd or and'd together, emit
2805 // this as a sequence of branches instead of setcc's with and/or operations.
2806 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2807 // unpredictable branches, and vector extracts because those jumps are likely
2808 // expensive for any target), this should improve performance.
2809 // For example, instead of something like:
2810 // cmp A, B
2811 // C = seteq
2812 // cmp D, E
2813 // F = setle
2814 // or C, F
2815 // jnz foo
2816 // Emit:
2817 // cmp A, B
2818 // je foo
2819 // cmp D, E
2820 // jle foo
2821 bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
2822 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2823 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2824 BOp->hasOneUse() && !IsUnpredictable) {
2825 Value *Vec;
2826 const Value *BOp0, *BOp1;
2828 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2829 Opcode = Instruction::And;
2830 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2831 Opcode = Instruction::Or;
2832
2833 if (Opcode &&
2834 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2835 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2837 FuncInfo, I, Opcode, BOp0, BOp1,
2839 Opcode, BOp0, BOp1))) {
2840 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2841 getEdgeProbability(BrMBB, Succ0MBB),
2842 getEdgeProbability(BrMBB, Succ1MBB),
2843 /*InvertCond=*/false);
2844 // If the compares in later blocks need to use values not currently
2845 // exported from this block, export them now. This block should always
2846 // be the first entry.
2847 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2848
2849 // Allow some cases to be rejected.
2850 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2851 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2852 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2853 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2854 }
2855
2856 // Emit the branch for this block.
2857 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2858 SL->SwitchCases.erase(SL->SwitchCases.begin());
2859 return;
2860 }
2861
2862 // Okay, we decided not to do this, remove any inserted MBB's and clear
2863 // SwitchCases.
2864 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2865 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2866
2867 SL->SwitchCases.clear();
2868 }
2869 }
2870
2871 // Create a CaseBlock record representing this branch.
2873 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2875 IsUnpredictable);
2876
2877 // Use visitSwitchCase to actually insert the fast branch sequence for this
2878 // cond branch.
2879 visitSwitchCase(CB, BrMBB);
2880}
2881
2882/// visitSwitchCase - Emits the necessary code to represent a single node in
2883/// the binary search tree resulting from lowering a switch instruction.
2885 MachineBasicBlock *SwitchBB) {
2886 SDValue Cond;
2887 SDValue CondLHS = getValue(CB.CmpLHS);
2888 SDLoc dl = CB.DL;
2889
2890 if (CB.CC == ISD::SETTRUE) {
2891 // Branch or fall through to TrueBB.
2892 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2893 SwitchBB->normalizeSuccProbs();
2894 if (CB.TrueBB != NextBlock(SwitchBB)) {
2895 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2896 DAG.getBasicBlock(CB.TrueBB)));
2897 }
2898 return;
2899 }
2900
2901 auto &TLI = DAG.getTargetLoweringInfo();
2902 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2903
2904 // Build the setcc now.
2905 if (!CB.CmpMHS) {
2906 // Fold "(X == true)" to X and "(X == false)" to !X to
2907 // handle common cases produced by branch lowering.
2908 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2909 CB.CC == ISD::SETEQ)
2910 Cond = CondLHS;
2911 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2912 CB.CC == ISD::SETEQ) {
2913 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2914 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2915 } else {
2916 SDValue CondRHS = getValue(CB.CmpRHS);
2917
2918 // If a pointer's DAG type is larger than its memory type then the DAG
2919 // values are zero-extended. This breaks signed comparisons so truncate
2920 // back to the underlying type before doing the compare.
2921 if (CondLHS.getValueType() != MemVT) {
2922 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2923 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2924 }
2925 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2926 }
2927 } else {
2928 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2929
2930 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2931 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2932
2933 SDValue CmpOp = getValue(CB.CmpMHS);
2934 EVT VT = CmpOp.getValueType();
2935
2936 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2937 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2938 ISD::SETLE);
2939 } else {
2940 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2941 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2942 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2943 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2944 }
2945 }
2946
2947 // Update successor info
2948 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2949 // TrueBB and FalseBB are always different unless the incoming IR is
2950 // degenerate. This only happens when running llc on weird IR.
2951 if (CB.TrueBB != CB.FalseBB)
2952 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2953 SwitchBB->normalizeSuccProbs();
2954
2955 // If the lhs block is the next block, invert the condition so that we can
2956 // fall through to the lhs instead of the rhs block.
2957 if (CB.TrueBB == NextBlock(SwitchBB)) {
2958 std::swap(CB.TrueBB, CB.FalseBB);
2959 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2960 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2961 }
2962
2963 SDNodeFlags Flags;
2964 Flags.setUnpredictable(CB.IsUnpredictable);
2965 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
2966 Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
2967
2968 setValue(CurInst, BrCond);
2969
2970 // Insert the false branch. Do this even if it's a fall through branch,
2971 // this makes it easier to do DAG optimizations which require inverting
2972 // the branch condition.
2973 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2975
2976 DAG.setRoot(BrCond);
2977}
2978
2979/// visitJumpTable - Emit JumpTable node in the current MBB
2981 // Emit the code for the jump table
2982 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2983 assert(JT.Reg != -1U && "Should lower JT Header first!");
2985 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
2986 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2987 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2988 Index.getValue(1), Table, Index);
2989 DAG.setRoot(BrJumpTable);
2990}
2991
2992/// visitJumpTableHeader - This function emits necessary code to produce index
2993/// in the JumpTable from switch case.
2995 JumpTableHeader &JTH,
2996 MachineBasicBlock *SwitchBB) {
2997 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2998 const SDLoc &dl = *JT.SL;
2999
3000 // Subtract the lowest switch case value from the value being switched on.
3001 SDValue SwitchOp = getValue(JTH.SValue);
3002 EVT VT = SwitchOp.getValueType();
3003 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
3004 DAG.getConstant(JTH.First, dl, VT));
3005
3006 // The SDNode we just created, which holds the value being switched on minus
3007 // the smallest case value, needs to be copied to a virtual register so it
3008 // can be used as an index into the jump table in a subsequent basic block.
3009 // This value may be smaller or larger than the target's pointer type, and
3010 // therefore require extension or truncating.
3012 SwitchOp =
3014
3015 unsigned JumpTableReg =
3017 SDValue CopyTo =
3018 DAG.getCopyToReg(getControlRoot(), dl, JumpTableReg, SwitchOp);
3019 JT.Reg = JumpTableReg;
3020
3021 if (!JTH.FallthroughUnreachable) {
3022 // Emit the range check for the jump table, and branch to the default block
3023 // for the switch statement if the value being switched on exceeds the
3024 // largest case in the switch.
3025 SDValue CMP = DAG.getSetCC(
3027 Sub.getValueType()),
3028 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3029
3030 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3031 MVT::Other, CopyTo, CMP,
3032 DAG.getBasicBlock(JT.Default));
3033
3034 // Avoid emitting unnecessary branches to the next block.
3035 if (JT.MBB != NextBlock(SwitchBB))
3036 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3037 DAG.getBasicBlock(JT.MBB));
3038
3039 DAG.setRoot(BrCond);
3040 } else {
3041 // Avoid emitting unnecessary branches to the next block.
3042 if (JT.MBB != NextBlock(SwitchBB))
3043 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3044 DAG.getBasicBlock(JT.MBB)));
3045 else
3046 DAG.setRoot(CopyTo);
3047 }
3048}
3049
3050/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3051/// variable if there exists one.
3053 SDValue &Chain) {
3054 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3055 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3056 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3060 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3061 if (Global) {
3062 MachinePointerInfo MPInfo(Global);
3066 MPInfo, Flags, LocationSize::precise(PtrTy.getSizeInBits() / 8),
3067 DAG.getEVTAlign(PtrTy));
3068 DAG.setNodeMemRefs(Node, {MemRef});
3069 }
3070 if (PtrTy != PtrMemTy)
3071 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3072 return SDValue(Node, 0);
3073}
3074
3075/// Codegen a new tail for a stack protector check ParentMBB which has had its
3076/// tail spliced into a stack protector check success bb.
3077///
3078/// For a high level explanation of how this fits into the stack protector
3079/// generation see the comment on the declaration of class
3080/// StackProtectorDescriptor.
3082 MachineBasicBlock *ParentBB) {
3083
3084 // First create the loads to the guard/stack slot for the comparison.
3086 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3087 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3088
3089 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3090 int FI = MFI.getStackProtectorIndex();
3091
3092 SDValue Guard;
3093 SDLoc dl = getCurSDLoc();
3094 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3095 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3096 Align Align =
3097 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
3098
3099 // Generate code to load the content of the guard slot.
3100 SDValue GuardVal = DAG.getLoad(
3101 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3104
3105 if (TLI.useStackGuardXorFP())
3106 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3107
3108 // Retrieve guard check function, nullptr if instrumentation is inlined.
3109 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3110 // The target provides a guard check function to validate the guard value.
3111 // Generate a call to that function with the content of the guard slot as
3112 // argument.
3113 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3114 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3115
3118 Entry.Node = GuardVal;
3119 Entry.Ty = FnTy->getParamType(0);
3120 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3121 Entry.IsInReg = true;
3122 Args.push_back(Entry);
3123
3127 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3128 getValue(GuardCheckFn), std::move(Args));
3129
3130 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3131 DAG.setRoot(Result.second);
3132 return;
3133 }
3134
3135 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3136 // Otherwise, emit a volatile load to retrieve the stack guard value.
3137 SDValue Chain = DAG.getEntryNode();
3138 if (TLI.useLoadStackGuardNode()) {
3139 Guard = getLoadStackGuard(DAG, dl, Chain);
3140 } else {
3141 const Value *IRGuard = TLI.getSDagStackGuard(M);
3142 SDValue GuardPtr = getValue(IRGuard);
3143
3144 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3145 MachinePointerInfo(IRGuard, 0), Align,
3147 }
3148
3149 // Perform the comparison via a getsetcc.
3151 *DAG.getContext(),
3152 Guard.getValueType()),
3153 Guard, GuardVal, ISD::SETNE);
3154
3155 // If the guard/stackslot do not equal, branch to failure MBB.
3156 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3157 MVT::Other, GuardVal.getOperand(0),
3158 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3159 // Otherwise branch to success MBB.
3160 SDValue Br = DAG.getNode(ISD::BR, dl,
3161 MVT::Other, BrCond,
3163
3164 DAG.setRoot(Br);
3165}
3166
3167/// Codegen the failure basic block for a stack protector check.
3168///
3169/// A failure stack protector machine basic block consists simply of a call to
3170/// __stack_chk_fail().
3171///
3172/// For a high level explanation of how this fits into the stack protector
3173/// generation see the comment on the declaration of class
3174/// StackProtectorDescriptor.
3175void
3179 CallOptions.setDiscardResult(true);
3180 SDValue Chain =
3181 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3182 std::nullopt, CallOptions, getCurSDLoc())
3183 .second;
3184 // On PS4/PS5, the "return address" must still be within the calling
3185 // function, even if it's at the very end, so emit an explicit TRAP here.
3186 // Passing 'true' for doesNotReturn above won't generate the trap for us.
3187 if (TM.getTargetTriple().isPS())
3188 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3189 // WebAssembly needs an unreachable instruction after a non-returning call,
3190 // because the function return type can be different from __stack_chk_fail's
3191 // return type (void).
3192 if (TM.getTargetTriple().isWasm())
3193 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3194
3195 DAG.setRoot(Chain);
3196}
3197
3198/// visitBitTestHeader - This function emits necessary code to produce value
3199/// suitable for "bit tests"
3201 MachineBasicBlock *SwitchBB) {
3202 SDLoc dl = getCurSDLoc();
3203
3204 // Subtract the minimum value.
3205 SDValue SwitchOp = getValue(B.SValue);
3206 EVT VT = SwitchOp.getValueType();
3207 SDValue RangeSub =
3208 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3209
3210 // Determine the type of the test operands.
3212 bool UsePtrType = false;
3213 if (!TLI.isTypeLegal(VT)) {
3214 UsePtrType = true;
3215 } else {
3216 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
3217 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
3218 // Switch table case range are encoded into series of masks.
3219 // Just use pointer type, it's guaranteed to fit.
3220 UsePtrType = true;
3221 break;
3222 }
3223 }
3224 SDValue Sub = RangeSub;
3225 if (UsePtrType) {
3226 VT = TLI.getPointerTy(DAG.getDataLayout());
3227 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3228 }
3229
3230 B.RegVT = VT.getSimpleVT();
3231 B.Reg = FuncInfo.CreateReg(B.RegVT);
3232 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3233
3234 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3235
3236 if (!B.FallthroughUnreachable)
3237 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3238 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3239 SwitchBB->normalizeSuccProbs();
3240
3241 SDValue Root = CopyTo;
3242 if (!B.FallthroughUnreachable) {
3243 // Conditional branch to the default block.
3244 SDValue RangeCmp = DAG.getSetCC(dl,
3246 RangeSub.getValueType()),
3247 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3248 ISD::SETUGT);
3249
3250 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3251 DAG.getBasicBlock(B.Default));
3252 }
3253
3254 // Avoid emitting unnecessary branches to the next block.
3255 if (MBB != NextBlock(SwitchBB))
3256 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3257
3258 DAG.setRoot(Root);
3259}
3260
3261/// visitBitTestCase - this function produces one "bit test"
3263 MachineBasicBlock* NextMBB,
3264 BranchProbability BranchProbToNext,
3265 unsigned Reg,
3266 BitTestCase &B,
3267 MachineBasicBlock *SwitchBB) {
3268 SDLoc dl = getCurSDLoc();
3269 MVT VT = BB.RegVT;
3270 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3271 SDValue Cmp;
3272 unsigned PopCount = llvm::popcount(B.Mask);
3274 if (PopCount == 1) {
3275 // Testing for a single bit; just compare the shift count with what it
3276 // would need to be to shift a 1 bit in that position.
3277 Cmp = DAG.getSetCC(
3279 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3280 ISD::SETEQ);
3281 } else if (PopCount == BB.Range) {
3282 // There is only one zero bit in the range, test for it directly.
3283 Cmp = DAG.getSetCC(
3285 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3286 } else {
3287 // Make desired shift
3288 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3289 DAG.getConstant(1, dl, VT), ShiftOp);
3290
3291 // Emit bit tests and jumps
3292 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3293 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3294 Cmp = DAG.getSetCC(
3296 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3297 }
3298
3299 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3300 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3301 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3302 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3303 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3304 // one as they are relative probabilities (and thus work more like weights),
3305 // and hence we need to normalize them to let the sum of them become one.
3306 SwitchBB->normalizeSuccProbs();
3307
3308 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3309 MVT::Other, getControlRoot(),
3310 Cmp, DAG.getBasicBlock(B.TargetBB));
3311
3312 // Avoid emitting unnecessary branches to the next block.
3313 if (NextMBB != NextBlock(SwitchBB))
3314 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3315 DAG.getBasicBlock(NextMBB));
3316
3317 DAG.setRoot(BrAnd);
3318}
3319
3320void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3321 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3322
3323 // Retrieve successors. Look through artificial IR level blocks like
3324 // catchswitch for successors.
3325 MachineBasicBlock *Return = FuncInfo.getMBB(I.getSuccessor(0));
3326 const BasicBlock *EHPadBB = I.getSuccessor(1);
3327 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(EHPadBB);
3328
3329 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3330 // have to do anything here to lower funclet bundles.
3331 assert(!I.hasOperandBundlesOtherThan(
3332 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3333 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3334 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3335 LLVMContext::OB_clang_arc_attachedcall}) &&
3336 "Cannot lower invokes with arbitrary operand bundles yet!");
3337
3338 const Value *Callee(I.getCalledOperand());
3339 const Function *Fn = dyn_cast<Function>(Callee);
3340 if (isa<InlineAsm>(Callee))
3341 visitInlineAsm(I, EHPadBB);
3342 else if (Fn && Fn->isIntrinsic()) {
3343 switch (Fn->getIntrinsicID()) {
3344 default:
3345 llvm_unreachable("Cannot invoke this intrinsic");
3346 case Intrinsic::donothing:
3347 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3348 case Intrinsic::seh_try_begin:
3349 case Intrinsic::seh_scope_begin:
3350 case Intrinsic::seh_try_end:
3351 case Intrinsic::seh_scope_end:
3352 if (EHPadMBB)
3353 // a block referenced by EH table
3354 // so dtor-funclet not removed by opts
3355 EHPadMBB->setMachineBlockAddressTaken();
3356 break;
3357 case Intrinsic::experimental_patchpoint_void:
3358 case Intrinsic::experimental_patchpoint:
3359 visitPatchpoint(I, EHPadBB);
3360 break;
3361 case Intrinsic::experimental_gc_statepoint:
3362 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3363 break;
3364 case Intrinsic::wasm_rethrow: {
3365 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3366 // special because it can be invoked, so we manually lower it to a DAG
3367 // node here.
3369 Ops.push_back(getControlRoot()); // inchain for the terminator node
3371 Ops.push_back(
3372 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3374 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3376 break;
3377 }
3378 }
3379 } else if (I.hasDeoptState()) {
3380 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3381 // Eventually we will support lowering the @llvm.experimental.deoptimize
3382 // intrinsic, and right now there are no plans to support other intrinsics
3383 // with deopt state.
3384 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3385 } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
3386 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), EHPadBB);
3387 } else {
3388 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3389 }
3390
3391 // If the value of the invoke is used outside of its defining block, make it
3392 // available as a virtual register.
3393 // We already took care of the exported value for the statepoint instruction
3394 // during call to the LowerStatepoint.
3395 if (!isa<GCStatepointInst>(I)) {
3397 }
3398
3401 BranchProbability EHPadBBProb =
3402 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3404 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3405
3406 // Update successor info.
3407 addSuccessorWithProb(InvokeMBB, Return);
3408 for (auto &UnwindDest : UnwindDests) {
3409 UnwindDest.first->setIsEHPad();
3410 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3411 }
3412 InvokeMBB->normalizeSuccProbs();
3413
3414 // Drop into normal successor.
3416 DAG.getBasicBlock(Return)));
3417}
3418
3419void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3420 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3421
3422 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3423 // have to do anything here to lower funclet bundles.
3424 assert(!I.hasOperandBundlesOtherThan(
3425 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3426 "Cannot lower callbrs with arbitrary operand bundles yet!");
3427
3428 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3429 visitInlineAsm(I);
3431
3432 // Retrieve successors.
3434 Dests.insert(I.getDefaultDest());
3435 MachineBasicBlock *Return = FuncInfo.getMBB(I.getDefaultDest());
3436
3437 // Update successor info.
3438 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3439 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3440 BasicBlock *Dest = I.getIndirectDest(i);
3442 Target->setIsInlineAsmBrIndirectTarget();
3443 Target->setMachineBlockAddressTaken();
3444 Target->setLabelMustBeEmitted();
3445 // Don't add duplicate machine successors.
3446 if (Dests.insert(Dest).second)
3447 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3448 }
3449 CallBrMBB->normalizeSuccProbs();
3450
3451 // Drop into default successor.
3453 MVT::Other, getControlRoot(),
3454 DAG.getBasicBlock(Return)));
3455}
3456
3457void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3458 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3459}
3460
3461void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3463 "Call to landingpad not in landing pad!");
3464
3465 // If there aren't registers to copy the values into (e.g., during SjLj
3466 // exceptions), then don't bother to create these DAG nodes.
3468 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3469 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3470 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3471 return;
3472
3473 // If landingpad's return type is token type, we don't create DAG nodes
3474 // for its exception pointer and selector value. The extraction of exception
3475 // pointer or selector value from token type landingpads is not currently
3476 // supported.
3477 if (LP.getType()->isTokenTy())
3478 return;
3479
3480 SmallVector<EVT, 2> ValueVTs;
3481 SDLoc dl = getCurSDLoc();
3482 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3483 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3484
3485 // Get the two live-in registers as SDValues. The physregs have already been
3486 // copied into virtual registers.
3487 SDValue Ops[2];
3489 Ops[0] = DAG.getZExtOrTrunc(
3493 dl, ValueVTs[0]);
3494 } else {
3495 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3496 }
3497 Ops[1] = DAG.getZExtOrTrunc(
3501 dl, ValueVTs[1]);
3502
3503 // Merge into one.
3505 DAG.getVTList(ValueVTs), Ops);
3506 setValue(&LP, Res);
3507}
3508
3511 // Update JTCases.
3512 for (JumpTableBlock &JTB : SL->JTCases)
3513 if (JTB.first.HeaderBB == First)
3514 JTB.first.HeaderBB = Last;
3515
3516 // Update BitTestCases.
3517 for (BitTestBlock &BTB : SL->BitTestCases)
3518 if (BTB.Parent == First)
3519 BTB.Parent = Last;
3520}
3521
3522void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3523 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3524
3525 // Update machine-CFG edges with unique successors.
3527 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3528 BasicBlock *BB = I.getSuccessor(i);
3529 bool Inserted = Done.insert(BB).second;
3530 if (!Inserted)
3531 continue;
3532
3533 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3534 addSuccessorWithProb(IndirectBrMBB, Succ);
3535 }
3536 IndirectBrMBB->normalizeSuccProbs();
3537
3539 MVT::Other, getControlRoot(),
3540 getValue(I.getAddress())));
3541}
3542
3543void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3545 return;
3546
3547 // We may be able to ignore unreachable behind a noreturn call.
3548 if (const CallInst *Call = dyn_cast_or_null<CallInst>(I.getPrevNode());
3549 Call && Call->doesNotReturn()) {
3551 return;
3552 // Do not emit an additional trap instruction.
3553 if (Call->isNonContinuableTrap())
3554 return;
3555 }
3556
3557 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3558}
3559
3560void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3562 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3563 Flags.copyFMF(*FPOp);
3564
3565 SDValue Op = getValue(I.getOperand(0));
3566 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3567 Op, Flags);
3568 setValue(&I, UnNodeValue);
3569}
3570
3571void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3573 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3574 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3575 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3576 }
3577 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3578 Flags.setExact(ExactOp->isExact());
3579 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3580 Flags.setDisjoint(DisjointOp->isDisjoint());
3581 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3582 Flags.copyFMF(*FPOp);
3583
3584 SDValue Op1 = getValue(I.getOperand(0));
3585 SDValue Op2 = getValue(I.getOperand(1));
3586 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3587 Op1, Op2, Flags);
3588 setValue(&I, BinNodeValue);
3589}
3590
3591void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3592 SDValue Op1 = getValue(I.getOperand(0));
3593 SDValue Op2 = getValue(I.getOperand(1));
3594
3596 Op1.getValueType(), DAG.getDataLayout());
3597
3598 // Coerce the shift amount to the right type if we can. This exposes the
3599 // truncate or zext to optimization early.
3600 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3602 "Unexpected shift type");
3603 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3604 }
3605
3606 bool nuw = false;
3607 bool nsw = false;
3608 bool exact = false;
3609
3610 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3611
3612 if (const OverflowingBinaryOperator *OFBinOp =
3613 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3614 nuw = OFBinOp->hasNoUnsignedWrap();
3615 nsw = OFBinOp->hasNoSignedWrap();
3616 }
3617 if (const PossiblyExactOperator *ExactOp =
3618 dyn_cast<const PossiblyExactOperator>(&I))
3619 exact = ExactOp->isExact();
3620 }
3622 Flags.setExact(exact);
3623 Flags.setNoSignedWrap(nsw);
3624 Flags.setNoUnsignedWrap(nuw);
3625 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3626 Flags);
3627 setValue(&I, Res);
3628}
3629
3630void SelectionDAGBuilder::visitSDiv(const User &I) {
3631 SDValue Op1 = getValue(I.getOperand(0));
3632 SDValue Op2 = getValue(I.getOperand(1));
3633
3635 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3636 cast<PossiblyExactOperator>(&I)->isExact());
3638 Op2, Flags));
3639}
3640
3641void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3642 ICmpInst::Predicate predicate = I.getPredicate();
3643 SDValue Op1 = getValue(I.getOperand(0));
3644 SDValue Op2 = getValue(I.getOperand(1));
3645 ISD::CondCode Opcode = getICmpCondCode(predicate);
3646
3647 auto &TLI = DAG.getTargetLoweringInfo();
3648 EVT MemVT =
3649 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3650
3651 // If a pointer's DAG type is larger than its memory type then the DAG values
3652 // are zero-extended. This breaks signed comparisons so truncate back to the
3653 // underlying type before doing the compare.
3654 if (Op1.getValueType() != MemVT) {
3655 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3656 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3657 }
3658
3660 I.getType());
3661 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3662}
3663
3664void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3665 FCmpInst::Predicate predicate = I.getPredicate();
3666 SDValue Op1 = getValue(I.getOperand(0));
3667 SDValue Op2 = getValue(I.getOperand(1));
3668
3669 ISD::CondCode Condition = getFCmpCondCode(predicate);
3670 auto *FPMO = cast<FPMathOperator>(&I);
3671 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3672 Condition = getFCmpCodeWithoutNaN(Condition);
3673
3675 Flags.copyFMF(*FPMO);
3676 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3677
3679 I.getType());
3680 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3681}
3682
3683// Check if the condition of the select has one use or two users that are both
3684// selects with the same condition.
3685static bool hasOnlySelectUsers(const Value *Cond) {
3686 return llvm::all_of(Cond->users(), [](const Value *V) {
3687 return isa<SelectInst>(V);
3688 });
3689}
3690
3691void SelectionDAGBuilder::visitSelect(const User &I) {
3692 SmallVector<EVT, 4> ValueVTs;
3694 ValueVTs);
3695 unsigned NumValues = ValueVTs.size();
3696 if (NumValues == 0) return;
3697
3698 SmallVector<SDValue, 4> Values(NumValues);
3699 SDValue Cond = getValue(I.getOperand(0));
3700 SDValue LHSVal = getValue(I.getOperand(1));
3701 SDValue RHSVal = getValue(I.getOperand(2));
3702 SmallVector<SDValue, 1> BaseOps(1, Cond);
3704 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3705
3706 bool IsUnaryAbs = false;
3707 bool Negate = false;
3708
3710 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3711 Flags.copyFMF(*FPOp);
3712
3713 Flags.setUnpredictable(
3714 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3715
3716 // Min/max matching is only viable if all output VTs are the same.
3717 if (all_equal(ValueVTs)) {
3718 EVT VT = ValueVTs[0];
3719 LLVMContext &Ctx = *DAG.getContext();
3720 auto &TLI = DAG.getTargetLoweringInfo();
3721
3722 // We care about the legality of the operation after it has been type
3723 // legalized.
3724 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3725 VT = TLI.getTypeToTransformTo(Ctx, VT);
3726
3727 // If the vselect is legal, assume we want to leave this as a vector setcc +
3728 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3729 // min/max is legal on the scalar type.
3730 bool UseScalarMinMax = VT.isVector() &&
3732
3733 // ValueTracking's select pattern matching does not account for -0.0,
3734 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3735 // -0.0 is less than +0.0.
3736 const Value *LHS, *RHS;
3737 auto SPR = matchSelectPattern(&I, LHS, RHS);
3739 switch (SPR.Flavor) {
3740 case SPF_UMAX: Opc = ISD::UMAX; break;
3741 case SPF_UMIN: Opc = ISD::UMIN; break;
3742 case SPF_SMAX: Opc = ISD::SMAX; break;
3743 case SPF_SMIN: Opc = ISD::SMIN; break;
3744 case SPF_FMINNUM:
3745 switch (SPR.NaNBehavior) {
3746 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3747 case SPNB_RETURNS_NAN: break;
3748 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3749 case SPNB_RETURNS_ANY:
3751 (UseScalarMinMax &&
3753 Opc = ISD::FMINNUM;
3754 break;
3755 }
3756 break;
3757 case SPF_FMAXNUM:
3758 switch (SPR.NaNBehavior) {
3759 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3760 case SPNB_RETURNS_NAN: break;
3761 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3762 case SPNB_RETURNS_ANY:
3764 (UseScalarMinMax &&
3766 Opc = ISD::FMAXNUM;
3767 break;
3768 }
3769 break;
3770 case SPF_NABS:
3771 Negate = true;
3772 [[fallthrough]];
3773 case SPF_ABS:
3774 IsUnaryAbs = true;
3775 Opc = ISD::ABS;
3776 break;
3777 default: break;
3778 }
3779
3780 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3781 (TLI.isOperationLegalOrCustomOrPromote(Opc, VT) ||
3782 (UseScalarMinMax &&
3783 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3784 // If the underlying comparison instruction is used by any other
3785 // instruction, the consumed instructions won't be destroyed, so it is
3786 // not profitable to convert to a min/max.
3787 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3788 OpCode = Opc;
3789 LHSVal = getValue(LHS);
3790 RHSVal = getValue(RHS);
3791 BaseOps.clear();
3792 }
3793
3794 if (IsUnaryAbs) {
3795 OpCode = Opc;
3796 LHSVal = getValue(LHS);
3797 BaseOps.clear();
3798 }
3799 }
3800
3801 if (IsUnaryAbs) {
3802 for (unsigned i = 0; i != NumValues; ++i) {
3803 SDLoc dl = getCurSDLoc();
3804 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3805 Values[i] =
3806 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3807 if (Negate)
3808 Values[i] = DAG.getNegative(Values[i], dl, VT);
3809 }
3810 } else {
3811 for (unsigned i = 0; i != NumValues; ++i) {
3812 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3813 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3814 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3815 Values[i] = DAG.getNode(
3816 OpCode, getCurSDLoc(),
3817 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3818 }
3819 }
3820
3822 DAG.getVTList(ValueVTs), Values));
3823}
3824
3825void SelectionDAGBuilder::visitTrunc(const User &I) {
3826 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3827 SDValue N = getValue(I.getOperand(0));
3829 I.getType());
3831}
3832
3833void SelectionDAGBuilder::visitZExt(const User &I) {
3834 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3835 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3836 SDValue N = getValue(I.getOperand(0));
3837 auto &TLI = DAG.getTargetLoweringInfo();
3838 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3839
3841 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3842 Flags.setNonNeg(PNI->hasNonNeg());
3843
3844 // Eagerly use nonneg information to canonicalize towards sign_extend if
3845 // that is the target's preference.
3846 // TODO: Let the target do this later.
3847 if (Flags.hasNonNeg() &&
3848 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
3850 return;
3851 }
3852
3853 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
3854}
3855
3856void SelectionDAGBuilder::visitSExt(const User &I) {
3857 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3858 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3859 SDValue N = getValue(I.getOperand(0));
3861 I.getType());
3863}
3864
3865void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3866 // FPTrunc is never a no-op cast, no need to check
3867 SDValue N = getValue(I.getOperand(0));
3868 SDLoc dl = getCurSDLoc();
3870 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3871 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3873 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3874}
3875
3876void SelectionDAGBuilder::visitFPExt(const User &I) {
3877 // FPExt is never a no-op cast, no need to check
3878 SDValue N = getValue(I.getOperand(0));
3880 I.getType());
3882}
3883
3884void SelectionDAGBuilder::visitFPToUI(const User &I) {
3885 // FPToUI is never a no-op cast, no need to check
3886 SDValue N = getValue(I.getOperand(0));
3888 I.getType());
3890}
3891
3892void SelectionDAGBuilder::visitFPToSI(const User &I) {
3893 // FPToSI is never a no-op cast, no need to check
3894 SDValue N = getValue(I.getOperand(0));
3896 I.getType());
3898}
3899
3900void SelectionDAGBuilder::visitUIToFP(const User &I) {
3901 // UIToFP is never a no-op cast, no need to check
3902 SDValue N = getValue(I.getOperand(0));
3904 I.getType());
3906 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3907 Flags.setNonNeg(PNI->hasNonNeg());
3908
3909 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
3910}
3911
3912void SelectionDAGBuilder::visitSIToFP(const User &I) {
3913 // SIToFP is never a no-op cast, no need to check
3914 SDValue N = getValue(I.getOperand(0));
3916 I.getType());
3918}
3919
3920void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3921 // What to do depends on the size of the integer and the size of the pointer.
3922 // We can either truncate, zero extend, or no-op, accordingly.
3923 SDValue N = getValue(I.getOperand(0));
3924 auto &TLI = DAG.getTargetLoweringInfo();
3926 I.getType());
3927 EVT PtrMemVT =
3928 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3929 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3930 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3931 setValue(&I, N);
3932}
3933
3934void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3935 // What to do depends on the size of the integer and the size of the pointer.
3936 // We can either truncate, zero extend, or no-op, accordingly.
3937 SDValue N = getValue(I.getOperand(0));
3938 auto &TLI = DAG.getTargetLoweringInfo();
3939 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3940 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3941 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3942 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3943 setValue(&I, N);
3944}
3945
3946void SelectionDAGBuilder::visitBitCast(const User &I) {
3947 SDValue N = getValue(I.getOperand(0));
3948 SDLoc dl = getCurSDLoc();
3950 I.getType());
3951
3952 // BitCast assures us that source and destination are the same size so this is
3953 // either a BITCAST or a no-op.
3954 if (DestVT != N.getValueType())
3956 DestVT, N)); // convert types.
3957 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3958 // might fold any kind of constant expression to an integer constant and that
3959 // is not what we are looking for. Only recognize a bitcast of a genuine
3960 // constant integer as an opaque constant.
3961 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3962 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3963 /*isOpaque*/true));
3964 else
3965 setValue(&I, N); // noop cast.
3966}
3967
3968void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3970 const Value *SV = I.getOperand(0);
3971 SDValue N = getValue(SV);
3972 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3973
3974 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3975 unsigned DestAS = I.getType()->getPointerAddressSpace();
3976
3977 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3978 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3979
3980 setValue(&I, N);
3981}
3982
3983void SelectionDAGBuilder::visitInsertElement(const User &I) {
3985 SDValue InVec = getValue(I.getOperand(0));
3986 SDValue InVal = getValue(I.getOperand(1));
3987 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3990 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3991 InVec, InVal, InIdx));
3992}
3993
3994void SelectionDAGBuilder::visitExtractElement(const User &I) {
3996 SDValue InVec = getValue(I.getOperand(0));
3997 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
4000 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4001 InVec, InIdx));
4002}
4003
4004void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4005 SDValue Src1 = getValue(I.getOperand(0));
4006 SDValue Src2 = getValue(I.getOperand(1));
4008 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
4009 Mask = SVI->getShuffleMask();
4010 else
4011 Mask = cast<ConstantExpr>(I).getShuffleMask();
4012 SDLoc DL = getCurSDLoc();
4014 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4015 EVT SrcVT = Src1.getValueType();
4016
4017 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
4018 VT.isScalableVector()) {
4019 // Canonical splat form of first element of first input vector.
4020 SDValue FirstElt =
4023 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4024 return;
4025 }
4026
4027 // For now, we only handle splats for scalable vectors.
4028 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4029 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4030 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4031
4032 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4033 unsigned MaskNumElts = Mask.size();
4034
4035 if (SrcNumElts == MaskNumElts) {
4036 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4037 return;
4038 }
4039
4040 // Normalize the shuffle vector since mask and vector length don't match.
4041 if (SrcNumElts < MaskNumElts) {
4042 // Mask is longer than the source vectors. We can use concatenate vector to
4043 // make the mask and vectors lengths match.
4044
4045 if (MaskNumElts % SrcNumElts == 0) {
4046 // Mask length is a multiple of the source vector length.
4047 // Check if the shuffle is some kind of concatenation of the input
4048 // vectors.
4049 unsigned NumConcat = MaskNumElts / SrcNumElts;
4050 bool IsConcat = true;
4051 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4052 for (unsigned i = 0; i != MaskNumElts; ++i) {
4053 int Idx = Mask[i];
4054 if (Idx < 0)
4055 continue;
4056 // Ensure the indices in each SrcVT sized piece are sequential and that
4057 // the same source is used for the whole piece.
4058 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4059 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4060 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4061 IsConcat = false;
4062 break;
4063 }
4064 // Remember which source this index came from.
4065 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4066 }
4067
4068 // The shuffle is concatenating multiple vectors together. Just emit
4069 // a CONCAT_VECTORS operation.
4070 if (IsConcat) {
4071 SmallVector<SDValue, 8> ConcatOps;
4072 for (auto Src : ConcatSrcs) {
4073 if (Src < 0)
4074 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4075 else if (Src == 0)
4076 ConcatOps.push_back(Src1);
4077 else
4078 ConcatOps.push_back(Src2);
4079 }
4080 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4081 return;
4082 }
4083 }
4084
4085 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4086 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4087 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4088 PaddedMaskNumElts);
4089
4090 // Pad both vectors with undefs to make them the same length as the mask.
4091 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4092
4093 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4094 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4095 MOps1[0] = Src1;
4096 MOps2[0] = Src2;
4097
4098 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4099 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4100
4101 // Readjust mask for new input vector length.
4102 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4103 for (unsigned i = 0; i != MaskNumElts; ++i) {
4104 int Idx = Mask[i];
4105 if (Idx >= (int)SrcNumElts)
4106 Idx -= SrcNumElts - PaddedMaskNumElts;
4107 MappedOps[i] = Idx;
4108 }
4109
4110 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4111
4112 // If the concatenated vector was padded, extract a subvector with the
4113 // correct number of elements.
4114 if (MaskNumElts != PaddedMaskNumElts)
4117
4118 setValue(&I, Result);
4119 return;
4120 }
4121
4122 if (SrcNumElts > MaskNumElts) {
4123 // Analyze the access pattern of the vector to see if we can extract
4124 // two subvectors and do the shuffle.
4125 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
4126 bool CanExtract = true;
4127 for (int Idx : Mask) {
4128 unsigned Input = 0;
4129 if (Idx < 0)
4130 continue;
4131
4132 if (Idx >= (int)SrcNumElts) {
4133 Input = 1;
4134 Idx -= SrcNumElts;
4135 }
4136
4137 // If all the indices come from the same MaskNumElts sized portion of
4138 // the sources we can use extract. Also make sure the extract wouldn't
4139 // extract past the end of the source.
4140 int NewStartIdx = alignDown(Idx, MaskNumElts);
4141 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4142 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4143 CanExtract = false;
4144 // Make sure we always update StartIdx as we use it to track if all
4145 // elements are undef.
4146 StartIdx[Input] = NewStartIdx;
4147 }
4148
4149 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4150 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4151 return;
4152 }
4153 if (CanExtract) {
4154 // Extract appropriate subvector and generate a vector shuffle
4155 for (unsigned Input = 0; Input < 2; ++Input) {
4156 SDValue &Src = Input == 0 ? Src1 : Src2;
4157 if (StartIdx[Input] < 0)
4158 Src = DAG.getUNDEF(VT);
4159 else {
4160 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4161 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4162 }
4163 }
4164
4165 // Calculate new mask.
4166 SmallVector<int, 8> MappedOps(Mask);
4167 for (int &Idx : MappedOps) {
4168 if (Idx >= (int)SrcNumElts)
4169 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4170 else if (Idx >= 0)
4171 Idx -= StartIdx[0];
4172 }
4173
4174 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4175 return;
4176 }
4177 }
4178
4179 // We can't use either concat vectors or extract subvectors so fall back to
4180 // replacing the shuffle with extract and build vector.
4181 // to insert and build vector.
4182 EVT EltVT = VT.getVectorElementType();
4184 for (int Idx : Mask) {
4185 SDValue Res;
4186
4187 if (Idx < 0) {
4188 Res = DAG.getUNDEF(EltVT);
4189 } else {
4190 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4191 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4192
4193 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4195 }
4196
4197 Ops.push_back(Res);
4198 }
4199
4200 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4201}
4202
4203void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4204 ArrayRef<unsigned> Indices = I.getIndices();
4205 const Value *Op0 = I.getOperand(0);
4206 const Value *Op1 = I.getOperand(1);
4207 Type *AggTy = I.getType();
4208 Type *ValTy = Op1->getType();
4209 bool IntoUndef = isa<UndefValue>(Op0);
4210 bool FromUndef = isa<UndefValue>(Op1);
4211
4212 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4213
4215 SmallVector<EVT, 4> AggValueVTs;
4216 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4217 SmallVector<EVT, 4> ValValueVTs;
4218 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4219
4220 unsigned NumAggValues = AggValueVTs.size();
4221 unsigned NumValValues = ValValueVTs.size();
4222 SmallVector<SDValue, 4> Values(NumAggValues);
4223
4224 // Ignore an insertvalue that produces an empty object
4225 if (!NumAggValues) {
4226 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4227 return;
4228 }
4229
4230 SDValue Agg = getValue(Op0);
4231 unsigned i = 0;
4232 // Copy the beginning value(s) from the original aggregate.
4233 for (; i != LinearIndex; ++i)
4234 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4235 SDValue(Agg.getNode(), Agg.getResNo() + i);
4236 // Copy values from the inserted value(s).
4237 if (NumValValues) {
4238 SDValue Val = getValue(Op1);
4239 for (; i != LinearIndex + NumValValues; ++i)
4240 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4241 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4242 }
4243 // Copy remaining value(s) from the original aggregate.
4244 for (; i != NumAggValues; ++i)
4245 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4246 SDValue(Agg.getNode(), Agg.getResNo() + i);
4247
4249 DAG.getVTList(AggValueVTs), Values));
4250}
4251
4252void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4253 ArrayRef<unsigned> Indices = I.getIndices();
4254 const Value *Op0 = I.getOperand(0);
4255 Type *AggTy = Op0->getType();
4256 Type *ValTy = I.getType();
4257 bool OutOfUndef = isa<UndefValue>(Op0);
4258
4259 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4260
4262 SmallVector<EVT, 4> ValValueVTs;
4263 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4264
4265 unsigned NumValValues = ValValueVTs.size();
4266
4267 // Ignore a extractvalue that produces an empty object
4268 if (!NumValValues) {
4269 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4270 return;
4271 }
4272
4273 SmallVector<SDValue, 4> Values(NumValValues);
4274
4275 SDValue Agg = getValue(Op0);
4276 // Copy out the selected value(s).
4277 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4278 Values[i - LinearIndex] =
4279 OutOfUndef ?
4280 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4281 SDValue(Agg.getNode(), Agg.getResNo() + i);
4282
4284 DAG.getVTList(ValValueVTs), Values));
4285}
4286
4287void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4288 Value *Op0 = I.getOperand(0);
4289 // Note that the pointer operand may be a vector of pointers. Take the scalar
4290 // element which holds a pointer.
4291 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4292 SDValue N = getValue(Op0);
4293 SDLoc dl = getCurSDLoc();
4294 auto &TLI = DAG.getTargetLoweringInfo();
4295 GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();
4296
4297 // Normalize Vector GEP - all scalar operands should be converted to the
4298 // splat vector.
4299 bool IsVectorGEP = I.getType()->isVectorTy();
4300 ElementCount VectorElementCount =
4301 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4303
4304 if (IsVectorGEP && !N.getValueType().isVector()) {
4306 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
4307 N = DAG.getSplat(VT, dl, N);
4308 }
4309
4310 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
4311 GTI != E; ++GTI) {
4312 const Value *Idx = GTI.getOperand();
4313 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4314 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4315 if (Field) {
4316 // N = N + Offset
4319
4320 // In an inbounds GEP with an offset that is nonnegative even when
4321 // interpreted as signed, assume there is no unsigned overflow.
4323 if (NW.hasNoUnsignedWrap() ||
4324 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4325 Flags.setNoUnsignedWrap(true);
4326
4327 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
4328 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
4329 }
4330 } else {
4331 // IdxSize is the width of the arithmetic according to IR semantics.
4332 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4333 // (and fix up the result later).
4334 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4335 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4336 TypeSize ElementSize =
4337 GTI.getSequentialElementStride(DAG.getDataLayout());
4338 // We intentionally mask away the high bits here; ElementSize may not
4339 // fit in IdxTy.
4340 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
4341 bool ElementScalable = ElementSize.isScalable();
4342
4343 // If this is a scalar constant or a splat vector of constants,
4344 // handle it quickly.
4345 const auto *C = dyn_cast<Constant>(Idx);
4346 if (C && isa<VectorType>(C->getType()))
4347 C = C->getSplatValue();
4348
4349 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4350 if (CI && CI->isZero())
4351 continue;
4352 if (CI && !ElementScalable) {
4353 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4355 SDValue OffsVal;
4356 if (IsVectorGEP)
4357 OffsVal = DAG.getConstant(
4358 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4359 else
4360 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4361
4362 // In an inbounds GEP with an offset that is nonnegative even when
4363 // interpreted as signed, assume there is no unsigned overflow.
4365 if (NW.hasNoUnsignedWrap() ||
4366 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4367 Flags.setNoUnsignedWrap(true);
4368
4369 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4370
4371 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
4372 continue;
4373 }
4374
4375 // N = N + Idx * ElementMul;
4376 SDValue IdxN = getValue(Idx);
4377
4378 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4380 VectorElementCount);
4381 IdxN = DAG.getSplat(VT, dl, IdxN);
4382 }
4383
4384 // If the index is smaller or larger than intptr_t, truncate or extend
4385 // it.
4386 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4387
4388 if (ElementScalable) {
4389 EVT VScaleTy = N.getValueType().getScalarType();
4390 SDValue VScale = DAG.getNode(
4391 ISD::VSCALE, dl, VScaleTy,
4392 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4393 if (IsVectorGEP)
4394 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4395 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
4396 } else {
4397 // If this is a multiply by a power of two, turn it into a shl
4398 // immediately. This is a very common case.
4399 if (ElementMul != 1) {
4400 if (ElementMul.isPowerOf2()) {
4401 unsigned Amt = ElementMul.logBase2();
4402 IdxN = DAG.getNode(ISD::SHL, dl,
4403 N.getValueType(), IdxN,
4404 DAG.getConstant(Amt, dl, IdxN.getValueType()));
4405 } else {
4406 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4407 IdxN.getValueType());
4408 IdxN = DAG.getNode(ISD::MUL, dl,
4409 N.getValueType(), IdxN, Scale);
4410 }
4411 }
4412 }
4413
4414 N = DAG.getNode(ISD::ADD, dl,
4415 N.getValueType(), N, IdxN);
4416 }
4417 }
4418
4419 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4420 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4421 if (IsVectorGEP) {
4422 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4423 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4424 }
4425
4426 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4427 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4428
4429 setValue(&I, N);
4430}
4431
4432void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4433 // If this is a fixed sized alloca in the entry block of the function,
4434 // allocate it statically on the stack.
4435 if (FuncInfo.StaticAllocaMap.count(&I))
4436 return; // getValue will auto-populate this.
4437
4438 SDLoc dl = getCurSDLoc();
4439 Type *Ty = I.getAllocatedType();
4441 auto &DL = DAG.getDataLayout();
4442 TypeSize TySize = DL.getTypeAllocSize(Ty);
4443 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4444
4445 SDValue AllocSize = getValue(I.getArraySize());
4446
4447 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4448 if (AllocSize.getValueType() != IntPtr)
4449 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4450
4451 if (TySize.isScalable())
4452 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4453 DAG.getVScale(dl, IntPtr,
4454 APInt(IntPtr.getScalarSizeInBits(),
4455 TySize.getKnownMinValue())));
4456 else {
4457 SDValue TySizeValue =
4459 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4460 DAG.getZExtOrTrunc(TySizeValue, dl, IntPtr));
4461 }
4462
4463 // Handle alignment. If the requested alignment is less than or equal to
4464 // the stack alignment, ignore it. If the size is greater than or equal to
4465 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4467 if (*Alignment <= StackAlign)
4468 Alignment = std::nullopt;
4469
4470 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4471 // Round the size of the allocation up to the stack alignment size
4472 // by add SA-1 to the size. This doesn't overflow because we're computing
4473 // an address inside an alloca.
4475 Flags.setNoUnsignedWrap(true);
4476 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4477 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4478
4479 // Mask out the low bits for alignment purposes.
4480 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4481 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
4482
4483 SDValue Ops[] = {
4484 getRoot(), AllocSize,
4485 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4486 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4487 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4488 setValue(&I, DSA);
4489 DAG.setRoot(DSA.getValue(1));
4490
4492}
4493
4494static const MDNode *getRangeMetadata(const Instruction &I) {
4495 // If !noundef is not present, then !range violation results in a poison
4496 // value rather than immediate undefined behavior. In theory, transferring
4497 // these annotations to SDAG is fine, but in practice there are key SDAG
4498 // transforms that are known not to be poison-safe, such as folding logical
4499 // and/or to bitwise and/or. For now, only transfer !range if !noundef is
4500 // also present.
4501 if (!I.hasMetadata(LLVMContext::MD_noundef))
4502 return nullptr;
4503 return I.getMetadata(LLVMContext::MD_range);
4504}
4505
4506static std::optional<ConstantRange> getRange(const Instruction &I) {
4507 if (const auto *CB = dyn_cast<CallBase>(&I)) {
4508 // see comment in getRangeMetadata about this check
4509 if (CB->hasRetAttr(Attribute::NoUndef))
4510 return CB->getRange();
4511 }
4512 if (const MDNode *Range = getRangeMetadata(I))
4514 return std::nullopt;
4515}
4516
4517void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4518 if (I.isAtomic())
4519 return visitAtomicLoad(I);
4520
4522 const Value *SV = I.getOperand(0);
4523 if (TLI.supportSwiftError()) {
4524 // Swifterror values can come from either a function parameter with
4525 // swifterror attribute or an alloca with swifterror attribute.
4526 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4527 if (Arg->hasSwiftErrorAttr())
4528 return visitLoadFromSwiftError(I);
4529 }
4530
4531 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4532 if (Alloca->isSwiftError())
4533 return visitLoadFromSwiftError(I);
4534 }
4535 }
4536
4537 SDValue Ptr = getValue(SV);
4538
4539 Type *Ty = I.getType();
4540 SmallVector<EVT, 4> ValueVTs, MemVTs;
4542 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4543 unsigned NumValues = ValueVTs.size();
4544 if (NumValues == 0)
4545 return;
4546
4547 Align Alignment = I.getAlign();
4548 AAMDNodes AAInfo = I.getAAMetadata();
4549 const MDNode *Ranges = getRangeMetadata(I);
4550 bool isVolatile = I.isVolatile();
4551 MachineMemOperand::Flags MMOFlags =
4553
4554 SDValue Root;
4555 bool ConstantMemory = false;
4556 if (isVolatile)
4557 // Serialize volatile loads with other side effects.
4558 Root = getRoot();
4559 else if (NumValues > MaxParallelChains)
4560 Root = getMemoryRoot();
4561 else if (AA &&
4563 SV,
4565 AAInfo))) {
4566 // Do not serialize (non-volatile) loads of constant memory with anything.
4567 Root = DAG.getEntryNode();
4568 ConstantMemory = true;
4570 } else {
4571 // Do not serialize non-volatile loads against each other.
4572 Root = DAG.getRoot();
4573 }
4574
4575 SDLoc dl = getCurSDLoc();
4576
4577 if (isVolatile)
4578 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4579
4580 SmallVector<SDValue, 4> Values(NumValues);
4581 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4582
4583 unsigned ChainI = 0;
4584 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4585 // Serializing loads here may result in excessive register pressure, and
4586 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4587 // could recover a bit by hoisting nodes upward in the chain by recognizing
4588 // they are side-effect free or do not alias. The optimizer should really
4589 // avoid this case by converting large object/array copies to llvm.memcpy
4590 // (MaxParallelChains should always remain as failsafe).
4591 if (ChainI == MaxParallelChains) {
4592 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4593 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4594 ArrayRef(Chains.data(), ChainI));
4595 Root = Chain;
4596 ChainI = 0;
4597 }
4598
4599 // TODO: MachinePointerInfo only supports a fixed length offset.
4600 MachinePointerInfo PtrInfo =
4601 !Offsets[i].isScalable() || Offsets[i].isZero()
4602 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4604
4605 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4606 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
4607 MMOFlags, AAInfo, Ranges);
4608 Chains[ChainI] = L.getValue(1);
4609
4610 if (MemVTs[i] != ValueVTs[i])
4611 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4612
4613 Values[i] = L;
4614 }
4615
4616 if (!ConstantMemory) {
4617 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4618 ArrayRef(Chains.data(), ChainI));
4619 if (isVolatile)
4620 DAG.setRoot(Chain);
4621 else
4622 PendingLoads.push_back(Chain);
4623 }
4624
4626 DAG.getVTList(ValueVTs), Values));
4627}
4628
4629void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4631 "call visitStoreToSwiftError when backend supports swifterror");
4632
4633 SmallVector<EVT, 4> ValueVTs;
4635 const Value *SrcV = I.getOperand(0);
4637 SrcV->getType(), ValueVTs, &Offsets, 0);
4638 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4639 "expect a single EVT for swifterror");
4640
4641 SDValue Src = getValue(SrcV);
4642 // Create a virtual register, then update the virtual register.
4643 Register VReg =
4644 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4645 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4646 // Chain can be getRoot or getControlRoot.
4647 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4648 SDValue(Src.getNode(), Src.getResNo()));
4649 DAG.setRoot(CopyNode);
4650}
4651
4652void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4654 "call visitLoadFromSwiftError when backend supports swifterror");
4655
4656 assert(!I.isVolatile() &&
4657 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4658 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4659 "Support volatile, non temporal, invariant for load_from_swift_error");
4660
4661 const Value *SV = I.getOperand(0);
4662 Type *Ty = I.getType();
4663 assert(
4664 (!AA ||
4667 I.getAAMetadata()))) &&
4668 "load_from_swift_error should not be constant memory");
4669
4670 SmallVector<EVT, 4> ValueVTs;
4673 ValueVTs, &Offsets, 0);
4674 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4675 "expect a single EVT for swifterror");
4676
4677 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4679 getRoot(), getCurSDLoc(),
4680 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4681
4682 setValue(&I, L);
4683}
4684
4685void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4686 if (I.isAtomic())
4687 return visitAtomicStore(I);
4688
4689 const Value *SrcV = I.getOperand(0);
4690 const Value *PtrV = I.getOperand(1);
4691
4693 if (TLI.supportSwiftError()) {
4694 // Swifterror values can come from either a function parameter with
4695 // swifterror attribute or an alloca with swifterror attribute.
4696 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4697 if (Arg->hasSwiftErrorAttr())
4698 return visitStoreToSwiftError(I);
4699 }
4700
4701 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4702 if (Alloca->isSwiftError())
4703 return visitStoreToSwiftError(I);
4704 }
4705 }
4706
4707 SmallVector<EVT, 4> ValueVTs, MemVTs;
4710 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4711 unsigned NumValues = ValueVTs.size();
4712 if (NumValues == 0)
4713 return;
4714
4715 // Get the lowered operands. Note that we do this after
4716 // checking if NumResults is zero, because with zero results
4717 // the operands won't have values in the map.
4718 SDValue Src = getValue(SrcV);
4719 SDValue Ptr = getValue(PtrV);
4720
4721 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4722 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4723 SDLoc dl = getCurSDLoc();
4724 Align Alignment = I.getAlign();
4725 AAMDNodes AAInfo = I.getAAMetadata();
4726
4727 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4728
4729 unsigned ChainI = 0;
4730 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4731 // See visitLoad comments.
4732 if (ChainI == MaxParallelChains) {
4733 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4734 ArrayRef(Chains.data(), ChainI));
4735 Root = Chain;
4736 ChainI = 0;
4737 }
4738
4739 // TODO: MachinePointerInfo only supports a fixed length offset.
4740 MachinePointerInfo PtrInfo =
4741 !Offsets[i].isScalable() || Offsets[i].isZero()
4742 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4744
4745 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4746 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4747 if (MemVTs[i] != ValueVTs[i])
4748 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4749 SDValue St =
4750 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4751 Chains[ChainI] = St;
4752 }
4753
4754 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4755 ArrayRef(Chains.data(), ChainI));
4756 setValue(&I, StoreNode);
4757 DAG.setRoot(StoreNode);
4758}
4759
4760void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4761 bool IsCompressing) {
4762 SDLoc sdl = getCurSDLoc();
4763
4764 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4765 Align &Alignment) {
4766 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4767 Src0 = I.getArgOperand(0);
4768 Ptr = I.getArgOperand(1);
4769 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getAlignValue();
4770 Mask = I.getArgOperand(3);
4771 };
4772 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4773 Align &Alignment) {
4774 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4775 Src0 = I.getArgOperand(0);
4776 Ptr = I.getArgOperand(1);
4777 Mask = I.getArgOperand(2);
4778 Alignment = I.getParamAlign(1).valueOrOne();
4779 };
4780
4781 Value *PtrOperand, *MaskOperand, *Src0Operand;
4782 Align Alignment;
4783 if (IsCompressing)
4784 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4785 else
4786 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4787
4788 SDValue Ptr = getValue(PtrOperand);
4789 SDValue Src0 = getValue(Src0Operand);
4790 SDValue Mask = getValue(MaskOperand);
4791 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4792
4793 EVT VT = Src0.getValueType();
4794
4795 auto MMOFlags = MachineMemOperand::MOStore;
4796 if (I.hasMetadata(LLVMContext::MD_nontemporal))
4798
4800 MachinePointerInfo(PtrOperand), MMOFlags,
4801 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4802
4803 const auto &TLI = DAG.getTargetLoweringInfo();
4804 const auto &TTI =
4805 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
4806 SDValue StoreNode =
4807 !IsCompressing &&
4808 TTI.hasConditionalLoadStoreForType(I.getArgOperand(0)->getType())
4809 ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0,
4810 Mask)
4811 : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask,
4812 VT, MMO, ISD::UNINDEXED, /*Truncating=*/false,
4813 IsCompressing);
4814 DAG.setRoot(StoreNode);
4815 setValue(&I, StoreNode);
4816}
4817
4818// Get a uniform base for the Gather/Scatter intrinsic.
4819// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4820// We try to represent it as a base pointer + vector of indices.
4821// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4822// The first operand of the GEP may be a single pointer or a vector of pointers
4823// Example:
4824// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4825// or
4826// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4827// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4828//
4829// When the first GEP operand is a single pointer - it is the uniform base we
4830// are looking for. If first operand of the GEP is a splat vector - we
4831// extract the splat value and use it as a uniform base.
4832// In all other cases the function returns 'false'.
4834 ISD::MemIndexType &IndexType, SDValue &Scale,
4835 SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4836 uint64_t ElemSize) {
4837 SelectionDAG& DAG = SDB->DAG;
4838 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4839 const DataLayout &DL = DAG.getDataLayout();
4840
4841 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4842
4843 // Handle splat constant pointer.
4844 if (auto *C = dyn_cast<Constant>(Ptr)) {
4845 C = C->getSplatValue();
4846 if (!C)
4847 return false;
4848
4849 Base = SDB->getValue(C);
4850
4851 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4852 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4853 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4854 IndexType = ISD::SIGNED_SCALED;
4855 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4856 return true;
4857 }
4858
4859 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4860 if (!GEP || GEP->getParent() != CurBB)
4861 return false;
4862
4863 if (GEP->getNumOperands() != 2)
4864 return false;
4865
4866 const Value *BasePtr = GEP->getPointerOperand();
4867 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4868
4869 // Make sure the base is scalar and the index is a vector.
4870 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4871 return false;
4872
4873 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4874 if (ScaleVal.isScalable())
4875 return false;
4876
4877 // Target may not support the required addressing mode.
4878 if (ScaleVal != 1 &&
4879 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
4880 return false;
4881
4882 Base = SDB->getValue(BasePtr);
4883 Index = SDB->getValue(IndexVal);
4884 IndexType = ISD::SIGNED_SCALED;
4885
4886 Scale =
4887 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4888 return true;
4889}
4890
4891void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4892 SDLoc sdl = getCurSDLoc();
4893
4894 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4895 const Value *Ptr = I.getArgOperand(1);
4896 SDValue Src0 = getValue(I.getArgOperand(0));
4897 SDValue Mask = getValue(I.getArgOperand(3));
4898 EVT VT = Src0.getValueType();
4899 Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4900 ->getMaybeAlignValue()
4901 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4903
4904 SDValue Base;
4905 SDValue Index;
4906 ISD::MemIndexType IndexType;
4907 SDValue Scale;
4908 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4909 I.getParent(), VT.getScalarStoreSize());
4910
4911 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4914 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4915 if (!UniformBase) {
4917 Index = getValue(Ptr);
4918 IndexType = ISD::SIGNED_SCALED;
4919 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4920 }
4921
4922 EVT IdxVT = Index.getValueType();
4923 EVT EltTy = IdxVT.getVectorElementType();
4924 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4925 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4926 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4927 }
4928
4929 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4930 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4931 Ops, MMO, IndexType, false);
4932 DAG.setRoot(Scatter);
4933 setValue(&I, Scatter);
4934}
4935
4936void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4937 SDLoc sdl = getCurSDLoc();
4938
4939 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4940 Align &Alignment) {
4941 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4942 Ptr = I.getArgOperand(0);
4943 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getAlignValue();
4944 Mask = I.getArgOperand(2);
4945 Src0 = I.getArgOperand(3);
4946 };
4947 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4948 Align &Alignment) {
4949 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4950 Ptr = I.getArgOperand(0);
4951 Alignment = I.getParamAlign(0).valueOrOne();
4952 Mask = I.getArgOperand(1);
4953 Src0 = I.getArgOperand(2);
4954 };
4955
4956 Value *PtrOperand, *MaskOperand, *Src0Operand;
4957 Align Alignment;
4958 if (IsExpanding)
4959 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4960 else
4961 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4962
4963 SDValue Ptr = getValue(PtrOperand);
4964 SDValue Src0 = getValue(Src0Operand);
4965 SDValue Mask = getValue(MaskOperand);
4966 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4967
4968 EVT VT = Src0.getValueType();
4969 AAMDNodes AAInfo = I.getAAMetadata();
4970 const MDNode *Ranges = getRangeMetadata(I);
4971
4972 // Do not serialize masked loads of constant memory with anything.
4973 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4974 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4975
4976 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4977
4978 auto MMOFlags = MachineMemOperand::MOLoad;
4979 if (I.hasMetadata(LLVMContext::MD_nontemporal))
4981
4983 MachinePointerInfo(PtrOperand), MMOFlags,
4984 LocationSize::beforeOrAfterPointer(), Alignment, AAInfo, Ranges);
4985
4986 const auto &TLI = DAG.getTargetLoweringInfo();
4987 const auto &TTI =
4988 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
4989 // The Load/Res may point to different values and both of them are output
4990 // variables.
4991 SDValue Load;
4992 SDValue Res;
4993 if (!IsExpanding &&
4995 Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask);
4996 else
4997 Res = Load =
4998 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4999 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5000 if (AddToChain)
5001 PendingLoads.push_back(Load.getValue(1));
5002 setValue(&I, Res);
5003}
5004
5005void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5006 SDLoc sdl = getCurSDLoc();
5007
5008 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
5009 const Value *Ptr = I.getArgOperand(0);
5010 SDValue Src0 = getValue(I.getArgOperand(3));
5011 SDValue Mask = getValue(I.getArgOperand(2));
5012
5014 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5015 Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
5016 ->getMaybeAlignValue()
5017 .value_or(DAG.getEVTAlign(VT.getScalarType()));
5018
5019 const MDNode *Ranges = getRangeMetadata(I);
5020
5021 SDValue Root = DAG.getRoot();
5022 SDValue Base;
5023 SDValue Index;
5024 ISD::MemIndexType IndexType;
5025 SDValue Scale;
5026 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
5027 I.getParent(), VT.getScalarStoreSize());
5028 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5031 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
5032 Ranges);
5033
5034 if (!UniformBase) {
5036 Index = getValue(Ptr);
5037 IndexType = ISD::SIGNED_SCALED;
5038 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5039 }
5040
5041 EVT IdxVT = Index.getValueType();
5042 EVT EltTy = IdxVT.getVectorElementType();
5043 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5044 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
5045 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5046 }
5047
5048 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5049 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
5050 Ops, MMO, IndexType, ISD::NON_EXTLOAD);
5051
5052 PendingLoads.push_back(Gather.getValue(1));
5053 setValue(&I, Gather);
5054}
5055
5056void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5057 SDLoc dl = getCurSDLoc();
5058 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5059 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5060 SyncScope::ID SSID = I.getSyncScopeID();
5061
5062 SDValue InChain = getRoot();
5063
5064 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
5065 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5066
5069
5072 MachinePointerInfo(I.getPointerOperand()), Flags,
5074 AAMDNodes(), nullptr, SSID, SuccessOrdering, FailureOrdering);
5075
5077 dl, MemVT, VTs, InChain,
5078 getValue(I.getPointerOperand()),
5079 getValue(I.getCompareOperand()),
5080 getValue(I.getNewValOperand()), MMO);
5081
5082 SDValue OutChain = L.getValue(2);
5083
5084 setValue(&I, L);
5085 DAG.setRoot(OutChain);
5086}
5087
5088void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5089 SDLoc dl = getCurSDLoc();
5091 switch (I.getOperation()) {
5092 default: llvm_unreachable("Unknown atomicrmw operation");
5110 break;
5113 break;
5114 }
5115 AtomicOrdering Ordering = I.getOrdering();
5116 SyncScope::ID SSID = I.getSyncScopeID();
5117
5118 SDValue InChain = getRoot();
5119
5120 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5123
5126 MachinePointerInfo(I.getPointerOperand()), Flags,
5128 AAMDNodes(), nullptr, SSID, Ordering);
5129
5130 SDValue L =
5131 DAG.getAtomic(NT, dl, MemVT, InChain,
5132 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5133 MMO);
5134
5135 SDValue OutChain = L.getValue(1);
5136
5137 setValue(&I, L);
5138 DAG.setRoot(OutChain);
5139}
5140
5141void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5142 SDLoc dl = getCurSDLoc();
5144 SDValue Ops[3];
5145 Ops[0] = getRoot();
5146 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5148 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5150 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5151 setValue(&I, N);
5152 DAG.setRoot(N);
5153}
5154
5155void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5156 SDLoc dl = getCurSDLoc();
5157 AtomicOrdering Order = I.getOrdering();
5158 SyncScope::ID SSID = I.getSyncScopeID();
5159
5160 SDValue InChain = getRoot();
5161
5163 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5164 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5165
5166 if (!TLI.supportsUnalignedAtomics() &&
5167 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5168 report_fatal_error("Cannot generate unaligned atomic load");
5169
5171
5173 MachinePointerInfo(I.getPointerOperand()), Flags,
5174 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5175 nullptr, SSID, Order);
5176
5177 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5178
5179 SDValue Ptr = getValue(I.getPointerOperand());
5180 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
5181 Ptr, MMO);
5182
5183 SDValue OutChain = L.getValue(1);
5184 if (MemVT != VT)
5185 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5186
5187 setValue(&I, L);
5188 DAG.setRoot(OutChain);
5189}
5190
5191void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5192 SDLoc dl = getCurSDLoc();
5193
5194 AtomicOrdering Ordering = I.getOrdering();
5195 SyncScope::ID SSID = I.getSyncScopeID();
5196
5197 SDValue InChain = getRoot();
5198
5200 EVT MemVT =
5201 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5202
5203 if (!TLI.supportsUnalignedAtomics() &&
5204 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5205 report_fatal_error("Cannot generate unaligned atomic store");
5206
5208
5211 MachinePointerInfo(I.getPointerOperand()), Flags,
5212 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5213 nullptr, SSID, Ordering);
5214
5215 SDValue Val = getValue(I.getValueOperand());
5216 if (Val.getValueType() != MemVT)
5217 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5218 SDValue Ptr = getValue(I.getPointerOperand());
5219
5220 SDValue OutChain =
5221 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5222
5223 setValue(&I, OutChain);
5224 DAG.setRoot(OutChain);
5225}
5226
5227/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5228/// node.
5229void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5230 unsigned Intrinsic) {
5231 // Ignore the callsite's attributes. A specific call site may be marked with
5232 // readnone, but the lowering code will expect the chain based on the
5233 // definition.
5234 const Function *F = I.getCalledFunction();
5235 bool HasChain = !F->doesNotAccessMemory();
5236 bool OnlyLoad =
5237 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5238
5239 // Build the operand list.
5241 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5242 if (OnlyLoad) {
5243 // We don't need to serialize loads against other loads.
5244 Ops.push_back(DAG.getRoot());
5245 } else {
5246 Ops.push_back(getRoot());
5247 }
5248 }
5249
5250 // Info is set by getTgtMemIntrinsic
5253 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
5255 Intrinsic);
5256
5257 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5258 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
5260 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
5262
5263 // Add all operands of the call to the operand list.
5264 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5265 const Value *Arg = I.getArgOperand(i);
5266 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5267 Ops.push_back(getValue(Arg));
5268 continue;
5269 }
5270
5271 // Use TargetConstant instead of a regular constant for immarg.
5272 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5273 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5274 assert(CI->getBitWidth() <= 64 &&
5275 "large intrinsic immediates not handled");
5276 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5277 } else {
5278 Ops.push_back(
5279 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5280 }
5281 }
5282
5283 SmallVector<EVT, 4> ValueVTs;
5284 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5285
5286 if (HasChain)
5287 ValueVTs.push_back(MVT::Other);
5288
5289 SDVTList VTs = DAG.getVTList(ValueVTs);
5290
5291 // Propagate fast-math-flags from IR to node(s).
5293 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5294 Flags.copyFMF(*FPMO);
5295 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5296
5297 // Create the node.
5299
5300 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5301 auto *Token = Bundle->Inputs[0].get();
5302 SDValue ConvControlToken = getValue(Token);
5303 assert(Ops.back().getValueType() != MVT::Glue &&
5304 "Did not expected another glue node here.");
5305 ConvControlToken =
5306 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5307 Ops.push_back(ConvControlToken);
5308 }
5309
5310 // In some cases, custom collection of operands from CallInst I may be needed.
5312 if (IsTgtIntrinsic) {
5313 // This is target intrinsic that touches memory
5314 //
5315 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
5316 // didn't yield anything useful.
5318 if (Info.ptrVal)
5319 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5320 else if (Info.fallbackAddressSpace)
5321 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5322 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
5323 Info.memVT, MPI, Info.align, Info.flags,
5324 Info.size, I.getAAMetadata());
5325 } else if (!HasChain) {
5327 } else if (!I.getType()->isVoidTy()) {
5329 } else {
5331 }
5332
5333 if (HasChain) {
5334 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
5335 if (OnlyLoad)
5336 PendingLoads.push_back(Chain);
5337 else
5338 DAG.setRoot(Chain);
5339 }
5340
5341 if (!I.getType()->isVoidTy()) {
5342 if (!isa<VectorType>(I.getType()))
5343 Result = lowerRangeToAssertZExt(DAG, I, Result);
5344
5345 MaybeAlign Alignment = I.getRetAlign();
5346
5347 // Insert `assertalign` node if there's an alignment.
5348 if (InsertAssertAlign && Alignment) {
5349 Result =
5350 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5351 }
5352 }
5353
5354 setValue(&I, Result);
5355}
5356
5357/// GetSignificand - Get the significand and build it into a floating-point
5358/// number with exponent of 1:
5359///
5360/// Op = (Op & 0x007fffff) | 0x3f800000;
5361///
5362/// where Op is the hexadecimal representation of floating point value.
5364 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5365 DAG.getConstant(0x007fffff, dl, MVT::i32));
5366 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5367 DAG.getConstant(0x3f800000, dl, MVT::i32));
5368 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5369}
5370
5371/// GetExponent - Get the exponent:
5372///
5373/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5374///
5375/// where Op is the hexadecimal representation of floating point value.
5377 const TargetLowering &TLI, const SDLoc &dl) {
5378 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5379 DAG.getConstant(0x7f800000, dl, MVT::i32));
5380 SDValue t1 = DAG.getNode(
5381 ISD::SRL, dl, MVT::i32, t0,
5382 DAG.getConstant(23, dl,
5383 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
5384 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5385 DAG.getConstant(127, dl, MVT::i32));
5386 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5387}
5388
5389/// getF32Constant - Get 32-bit floating point constant.
5391 const SDLoc &dl) {
5392 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5393 MVT::f32);
5394}
5395
5397 SelectionDAG &DAG) {
5398 // TODO: What fast-math-flags should be set on the floating-point nodes?
5399
5400 // IntegerPartOfX = ((int32_t)(t0);
5401 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5402
5403 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5404 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5405 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5406
5407 // IntegerPartOfX <<= 23;
5408 IntegerPartOfX =
5409 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5410 DAG.getConstant(23, dl,
5412 MVT::i32, DAG.getDataLayout())));
5413
5414 SDValue TwoToFractionalPartOfX;
5415 if (LimitFloatPrecision <= 6) {
5416 // For floating-point precision of 6:
5417 //
5418 // TwoToFractionalPartOfX =
5419 // 0.997535578f +
5420 // (0.735607626f + 0.252464424f * x) * x;
5421 //
5422 // error 0.0144103317, which is 6 bits
5423 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5424 getF32Constant(DAG, 0x3e814304, dl));
5425 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5426 getF32Constant(DAG, 0x3f3c50c8, dl));
5427 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5428 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5429 getF32Constant(DAG, 0x3f7f5e7e, dl));
5430 } else if (LimitFloatPrecision <= 12) {
5431 // For floating-point precision of 12:
5432 //
5433 // TwoToFractionalPartOfX =
5434 // 0.999892986f +
5435 // (0.696457318f +
5436 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5437 //
5438 // error 0.000107046256, which is 13 to 14 bits
5439 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5440 getF32Constant(DAG, 0x3da235e3, dl));
5441 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5442 getF32Constant(DAG, 0x3e65b8f3, dl));
5443 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5444 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5445 getF32Constant(DAG, 0x3f324b07, dl));
5446 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5447 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5448 getF32Constant(DAG, 0x3f7ff8fd, dl));
5449 } else { // LimitFloatPrecision <= 18
5450 // For floating-point precision of 18:
5451 //
5452 // TwoToFractionalPartOfX =
5453 // 0.999999982f +
5454 // (0.693148872f +
5455 // (0.240227044f +
5456 // (0.554906021e-1f +
5457 // (0.961591928e-2f +
5458 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5459 // error 2.47208000*10^(-7), which is better than 18 bits
5460 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5461 getF32Constant(DAG, 0x3924b03e, dl));
5462 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5463 getF32Constant(DAG, 0x3ab24b87, dl));
5464 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5465 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5466 getF32Constant(DAG, 0x3c1d8c17, dl));
5467 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5468 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5469 getF32Constant(DAG, 0x3d634a1d, dl));
5470 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5471 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5472 getF32Constant(DAG, 0x3e75fe14, dl));
5473 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5474 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5475 getF32Constant(DAG, 0x3f317234, dl));
5476 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5477 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5478 getF32Constant(DAG, 0x3f800000, dl));
5479 }
5480
5481 // Add the exponent into the result in integer domain.
5482 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5483 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5484 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5485}
5486
5487/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5488/// limited-precision mode.
5490 const TargetLowering &TLI, SDNodeFlags Flags) {
5491 if (Op.getValueType() == MVT::f32 &&
5493
5494 // Put the exponent in the right bit position for later addition to the
5495 // final result:
5496 //
5497 // t0 = Op * log2(e)
5498
5499 // TODO: What fast-math-flags should be set here?
5500 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5501 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5502 return getLimitedPrecisionExp2(t0, dl, DAG);
5503 }
5504
5505 // No special expansion.
5506 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5507}
5508
5509/// expandLog - Lower a log intrinsic. Handles the special sequences for
5510/// limited-precision mode.
5512 const TargetLowering &TLI, SDNodeFlags Flags) {
5513 // TODO: What fast-math-flags should be set on the floating-point nodes?
5514
5515 if (Op.getValueType() == MVT::f32 &&
5517 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5518
5519 // Scale the exponent by log(2).
5520 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5521 SDValue LogOfExponent =
5522 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5523 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5524
5525 // Get the significand and build it into a floating-point number with
5526 // exponent of 1.
5527 SDValue X = GetSignificand(DAG, Op1, dl);
5528
5529 SDValue LogOfMantissa;
5530 if (LimitFloatPrecision <= 6) {
5531 // For floating-point precision of 6:
5532 //
5533 // LogofMantissa =
5534 // -1.1609546f +
5535 // (1.4034025f - 0.23903021f * x) * x;
5536 //
5537 // error 0.0034276066, which is better than 8 bits
5538 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5539 getF32Constant(DAG, 0xbe74c456, dl));
5540 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5541 getF32Constant(DAG, 0x3fb3a2b1, dl));
5542 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5543 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5544 getF32Constant(DAG, 0x3f949a29, dl));
5545 } else if (LimitFloatPrecision <= 12) {
5546 // For floating-point precision of 12:
5547 //
5548 // LogOfMantissa =
5549 // -1.7417939f +
5550 // (2.8212026f +
5551 // (-1.4699568f +
5552 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5553 //
5554 // error 0.000061011436, which is 14 bits
5555 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5556 getF32Constant(DAG, 0xbd67b6d6, dl));
5557 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5558 getF32Constant(DAG, 0x3ee4f4b8, dl));
5559 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5560 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5561 getF32Constant(DAG, 0x3fbc278b, dl));
5562 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5563 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5564 getF32Constant(DAG, 0x40348e95, dl));
5565 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5566 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5567 getF32Constant(DAG, 0x3fdef31a, dl));
5568 } else { // LimitFloatPrecision <= 18
5569 // For floating-point precision of 18:
5570 //
5571 // LogOfMantissa =
5572 // -2.1072184f +
5573 // (4.2372794f +
5574 // (-3.7029485f +
5575 // (2.2781945f +
5576 // (-0.87823314f +
5577 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5578 //
5579 // error 0.0000023660568, which is better than 18 bits
5580 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5581 getF32Constant(DAG, 0xbc91e5ac, dl));
5582 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5583 getF32Constant(DAG, 0x3e4350aa, dl));
5584 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5585 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5586 getF32Constant(DAG, 0x3f60d3e3, dl));
5587 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5588 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5589 getF32Constant(DAG, 0x4011cdf0, dl));
5590 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5591 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5592 getF32Constant(DAG, 0x406cfd1c, dl));
5593 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5594 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5595 getF32Constant(DAG, 0x408797cb, dl));
5596 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5597 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5598 getF32Constant(DAG, 0x4006dcab, dl));
5599 }
5600
5601 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5602 }
5603
5604 // No special expansion.
5605 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5606}
5607
5608/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5609/// limited-precision mode.
5611 const TargetLowering &TLI, SDNodeFlags Flags) {
5612 // TODO: What fast-math-flags should be set on the floating-point nodes?
5613
5614 if (Op.getValueType() == MVT::f32 &&
5616 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5617
5618 // Get the exponent.
5619 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5620
5621 // Get the significand and build it into a floating-point number with
5622 // exponent of 1.
5623 SDValue X = GetSignificand(DAG, Op1, dl);
5624
5625 // Different possible minimax approximations of significand in
5626 // floating-point for various degrees of accuracy over [1,2].
5627 SDValue Log2ofMantissa;
5628 if (LimitFloatPrecision <= 6) {
5629 // For floating-point precision of 6:
5630 //
5631 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5632 //
5633 // error 0.0049451742, which is more than 7 bits
5634 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5635 getF32Constant(DAG, 0xbeb08fe0, dl));
5636 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5637 getF32Constant(DAG, 0x40019463, dl));
5638 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5639 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5640 getF32Constant(DAG, 0x3fd6633d, dl));
5641 } else if (LimitFloatPrecision <= 12) {
5642 // For floating-point precision of 12:
5643 //
5644 // Log2ofMantissa =
5645 // -2.51285454f +
5646 // (4.07009056f +
5647 // (-2.12067489f +
5648 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5649 //
5650 // error 0.0000876136000, which is better than 13 bits
5651 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5652 getF32Constant(DAG, 0xbda7262e, dl));
5653 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5654 getF32Constant(DAG, 0x3f25280b, dl));
5655 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5656 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5657 getF32Constant(DAG, 0x4007b923, dl));
5658 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5659 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5660 getF32Constant(DAG, 0x40823e2f, dl));
5661 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5662 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5663 getF32Constant(DAG, 0x4020d29c, dl));
5664 } else { // LimitFloatPrecision <= 18
5665 // For floating-point precision of 18:
5666 //
5667 // Log2ofMantissa =
5668 // -3.0400495f +
5669 // (6.1129976f +
5670 // (-5.3420409f +
5671 // (3.2865683f +
5672 // (-1.2669343f +
5673 // (0.27515199f -
5674 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5675 //
5676 // error 0.0000018516, which is better than 18 bits
5677 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5678 getF32Constant(DAG, 0xbcd2769e, dl));
5679 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5680 getF32Constant(DAG, 0x3e8ce0b9, dl));
5681 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5682 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5683 getF32Constant(DAG, 0x3fa22ae7, dl));
5684 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5685 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5686 getF32Constant(DAG, 0x40525723, dl));
5687 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5688 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5689 getF32Constant(DAG, 0x40aaf200, dl));
5690 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5691 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5692 getF32Constant(DAG, 0x40c39dad, dl));
5693 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5694 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5695 getF32Constant(DAG, 0x4042902c, dl));
5696 }
5697
5698 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5699 }
5700
5701 // No special expansion.
5702 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5703}
5704
5705/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5706/// limited-precision mode.
5708 const TargetLowering &TLI, SDNodeFlags Flags) {
5709 // TODO: What fast-math-flags should be set on the floating-point nodes?
5710
5711 if (Op.getValueType() == MVT::f32 &&
5713 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5714
5715 // Scale the exponent by log10(2) [0.30102999f].
5716 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5717 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5718 getF32Constant(DAG, 0x3e9a209a, dl));
5719
5720 // Get the significand and build it into a floating-point number with
5721 // exponent of 1.
5722 SDValue X = GetSignificand(DAG, Op1, dl);
5723
5724 SDValue Log10ofMantissa;
5725 if (LimitFloatPrecision <= 6) {
5726 // For floating-point precision of 6:
5727 //
5728 // Log10ofMantissa =
5729 // -0.50419619f +
5730 // (0.60948995f - 0.10380950f * x) * x;
5731 //
5732 // error 0.0014886165, which is 6 bits
5733 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5734 getF32Constant(DAG, 0xbdd49a13, dl));
5735 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5736 getF32Constant(DAG, 0x3f1c0789, dl));
5737 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5738 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5739 getF32Constant(DAG, 0x3f011300, dl));
5740 } else if (LimitFloatPrecision <= 12) {
5741 // For floating-point precision of 12:
5742 //
5743 // Log10ofMantissa =
5744 // -0.64831180f +
5745 // (0.91751397f +
5746 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5747 //
5748 // error 0.00019228036, which is better than 12 bits
5749 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5750 getF32Constant(DAG, 0x3d431f31, dl));
5751 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5752 getF32Constant(DAG, 0x3ea21fb2, dl));
5753 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5754 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5755 getF32Constant(DAG, 0x3f6ae232, dl));
5756 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5757 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5758 getF32Constant(DAG, 0x3f25f7c3, dl));
5759 } else { // LimitFloatPrecision <= 18
5760 // For floating-point precision of 18:
5761 //
5762 // Log10ofMantissa =
5763 // -0.84299375f +
5764 // (1.5327582f +
5765 // (-1.0688956f +
5766 // (0.49102474f +
5767 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5768 //
5769 // error 0.0000037995730, which is better than 18 bits
5770 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5771 getF32Constant(DAG, 0x3c5d51ce, dl));
5772 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5773 getF32Constant(DAG, 0x3e00685a, dl));
5774 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5775 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5776 getF32Constant(DAG, 0x3efb6798, dl));
5777 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5778 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5779 getF32Constant(DAG, 0x3f88d192, dl));
5780 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5781 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5782 getF32Constant(DAG, 0x3fc4316c, dl));
5783 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5784 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5785 getF32Constant(DAG, 0x3f57ce70, dl));
5786 }
5787
5788 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5789 }
5790
5791 // No special expansion.
5792 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5793}
5794
5795/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5796/// limited-precision mode.
5798 const TargetLowering &TLI, SDNodeFlags Flags) {
5799 if (Op.getValueType() == MVT::f32 &&
5801 return getLimitedPrecisionExp2(Op, dl, DAG);
5802
5803 // No special expansion.
5804 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5805}
5806
5807/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5808/// limited-precision mode with x == 10.0f.
5809static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5810 SelectionDAG &DAG, const TargetLowering &TLI,
5811 SDNodeFlags Flags) {
5812 bool IsExp10 = false;
5813 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5815 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5816 APFloat Ten(10.0f);
5817 IsExp10 = LHSC->isExactlyValue(Ten);
5818 }
5819 }
5820
5821 // TODO: What fast-math-flags should be set on the FMUL node?
5822 if (IsExp10) {
5823 // Put the exponent in the right bit position for later addition to the
5824 // final result:
5825 //
5826 // #define LOG2OF10 3.3219281f
5827 // t0 = Op * LOG2OF10;
5828 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5829 getF32Constant(DAG, 0x40549a78, dl));
5830 return getLimitedPrecisionExp2(t0, dl, DAG);
5831 }
5832
5833 // No special expansion.
5834 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5835}
5836
5837/// ExpandPowI - Expand a llvm.powi intrinsic.
5838static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5839 SelectionDAG &DAG) {
5840 // If RHS is a constant, we can expand this out to a multiplication tree if
5841 // it's beneficial on the target, otherwise we end up lowering to a call to
5842 // __powidf2 (for example).
5843 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5844 unsigned Val = RHSC->getSExtValue();
5845
5846 // powi(x, 0) -> 1.0
5847 if (Val == 0)
5848 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5849
5851 Val, DAG.shouldOptForSize())) {
5852 // Get the exponent as a positive value.
5853 if ((int)Val < 0)
5854 Val = -Val;
5855 // We use the simple binary decomposition method to generate the multiply
5856 // sequence. There are more optimal ways to do this (for example,
5857 // powi(x,15) generates one more multiply than it should), but this has
5858 // the benefit of being both really simple and much better than a libcall.
5859 SDValue Res; // Logically starts equal to 1.0
5860 SDValue CurSquare = LHS;
5861 // TODO: Intrinsics should have fast-math-flags that propagate to these
5862 // nodes.
5863 while (Val) {
5864 if (Val & 1) {
5865 if (Res.getNode())
5866 Res =
5867 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5868 else
5869 Res = CurSquare; // 1.0*CurSquare.
5870 }
5871
5872 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5873 CurSquare, CurSquare);
5874 Val >>= 1;
5875 }
5876
5877 // If the original was negative, invert the result, producing 1/(x*x*x).
5878 if (RHSC->getSExtValue() < 0)
5879 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5880 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5881 return Res;
5882 }
5883 }
5884
5885 // Otherwise, expand to a libcall.
5886 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5887}
5888
5889static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5890 SDValue LHS, SDValue RHS, SDValue Scale,
5891 SelectionDAG &DAG, const TargetLowering &TLI) {
5892 EVT VT = LHS.getValueType();
5893 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5894 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5895 LLVMContext &Ctx = *DAG.getContext();
5896
5897 // If the type is legal but the operation isn't, this node might survive all
5898 // the way to operation legalization. If we end up there and we do not have
5899 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5900 // node.
5901
5902 // Coax the legalizer into expanding the node during type legalization instead
5903 // by bumping the size by one bit. This will force it to Promote, enabling the
5904 // early expansion and avoiding the need to expand later.
5905
5906 // We don't have to do this if Scale is 0; that can always be expanded, unless
5907 // it's a saturating signed operation. Those can experience true integer
5908 // division overflow, a case which we must avoid.
5909
5910 // FIXME: We wouldn't have to do this (or any of the early
5911 // expansion/promotion) if it was possible to expand a libcall of an
5912 // illegal type during operation legalization. But it's not, so things
5913 // get a bit hacky.
5914 unsigned ScaleInt = Scale->getAsZExtVal();
5915 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5916 (TLI.isTypeLegal(VT) ||
5917 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5919 Opcode, VT, ScaleInt);
5920 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5921 EVT PromVT;
5922 if (VT.isScalarInteger())
5923 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5924 else if (VT.isVector()) {
5925 PromVT = VT.getVectorElementType();
5926 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5927 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5928 } else
5929 llvm_unreachable("Wrong VT for DIVFIX?");
5930 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
5931 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
5932 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5933 // For saturating operations, we need to shift up the LHS to get the
5934 // proper saturation width, and then shift down again afterwards.
5935 if (Saturating)
5936 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5937 DAG.getConstant(1, DL, ShiftTy));
5938 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5939 if (Saturating)
5940 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5941 DAG.getConstant(1, DL, ShiftTy));
5942 return DAG.getZExtOrTrunc(Res, DL, VT);
5943 }
5944 }
5945
5946 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5947}
5948
5949// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5950// bitcasted, or split argument. Returns a list of <Register, size in bits>
5951static void
5952getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5953 const SDValue &N) {
5954 switch (N.getOpcode()) {
5955 case ISD::CopyFromReg: {
5956 SDValue Op = N.getOperand(1);
5957 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5958 Op.getValueType().getSizeInBits());
5959 return;
5960 }
5961 case ISD::BITCAST:
5962 case ISD::AssertZext:
5963 case ISD::AssertSext:
5964 case ISD::TRUNCATE:
5965 getUnderlyingArgRegs(Regs, N.getOperand(0));
5966 return;
5967 case ISD::BUILD_PAIR:
5968 case ISD::BUILD_VECTOR:
5970 for (SDValue Op : N->op_values())
5971 getUnderlyingArgRegs(Regs, Op);
5972 return;
5973 default:
5974 return;
5975 }
5976}
5977
5978/// If the DbgValueInst is a dbg_value of a function argument, create the
5979/// corresponding DBG_VALUE machine instruction for it now. At the end of
5980/// instruction selection, they will be inserted to the entry BB.
5981/// We don't currently support this for variadic dbg_values, as they shouldn't
5982/// appear for function arguments or in the prologue.
5983bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5984 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5985 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5986 const Argument *Arg = dyn_cast<Argument>(V);
5987 if (!Arg)
5988 return false;
5989
5992
5993 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5994 // we've been asked to pursue.
5995 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5996 bool Indirect) {
5997 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5998 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5999 // pointing at the VReg, which will be patched up later.
6000 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
6002 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6003 /* isKill */ false, /* isDead */ false,
6004 /* isUndef */ false, /* isEarlyClobber */ false,
6005 /* SubReg */ 0, /* isDebug */ true)});
6006
6007 auto *NewDIExpr = FragExpr;
6008 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6009 // the DIExpression.
6010 if (Indirect)
6011 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
6013 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
6014 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
6015 } else {
6016 // Create a completely standard DBG_VALUE.
6017 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
6018 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
6019 }
6020 };
6021
6022 if (Kind == FuncArgumentDbgValueKind::Value) {
6023 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6024 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6025 // the entry block.
6026 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6027 if (!IsInEntryBlock)
6028 return false;
6029
6030 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6031 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6032 // variable that also is a param.
6033 //
6034 // Although, if we are at the top of the entry block already, we can still
6035 // emit using ArgDbgValue. This might catch some situations when the
6036 // dbg.value refers to an argument that isn't used in the entry block, so
6037 // any CopyToReg node would be optimized out and the only way to express
6038 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6039 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6040 // we should only emit as ArgDbgValue if the Variable is an argument to the
6041 // current function, and the dbg.value intrinsic is found in the entry
6042 // block.
6043 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6044 !DL->getInlinedAt();
6045 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6046 if (!IsInPrologue && !VariableIsFunctionInputArg)
6047 return false;
6048
6049 // Here we assume that a function argument on IR level only can be used to
6050 // describe one input parameter on source level. If we for example have
6051 // source code like this
6052 //
6053 // struct A { long x, y; };
6054 // void foo(struct A a, long b) {
6055 // ...
6056 // b = a.x;
6057 // ...
6058 // }
6059 //
6060 // and IR like this
6061 //
6062 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6063 // entry:
6064 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6065 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6066 // call void @llvm.dbg.value(metadata i32 %b, "b",
6067 // ...
6068 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6069 // ...
6070 //
6071 // then the last dbg.value is describing a parameter "b" using a value that
6072 // is an argument. But since we already has used %a1 to describe a parameter
6073 // we should not handle that last dbg.value here (that would result in an
6074 // incorrect hoisting of the DBG_VALUE to the function entry).
6075 // Notice that we allow one dbg.value per IR level argument, to accommodate
6076 // for the situation with fragments above.
6077 // If there is no node for the value being handled, we return true to skip
6078 // the normal generation of debug info, as it would kill existing debug
6079 // info for the parameter in case of duplicates.
6080 if (VariableIsFunctionInputArg) {
6081 unsigned ArgNo = Arg->getArgNo();
6082 if (ArgNo >= FuncInfo.DescribedArgs.size())
6083 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6084 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6085 return !NodeMap[V].getNode();
6086 FuncInfo.DescribedArgs.set(ArgNo);
6087 }
6088 }
6089
6090 bool IsIndirect = false;
6091 std::optional<MachineOperand> Op;
6092 // Some arguments' frame index is recorded during argument lowering.
6093 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6094 if (FI != std::numeric_limits<int>::max())
6096
6098 if (!Op && N.getNode()) {
6099 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6100 Register Reg;
6101 if (ArgRegsAndSizes.size() == 1)
6102 Reg = ArgRegsAndSizes.front().first;
6103
6104 if (Reg && Reg.isVirtual()) {
6106 Register PR = RegInfo.getLiveInPhysReg(Reg);
6107 if (PR)
6108 Reg = PR;
6109 }
6110 if (Reg) {
6111 Op = MachineOperand::CreateReg(Reg, false);
6112 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6113 }
6114 }
6115
6116 if (!Op && N.getNode()) {
6117 // Check if frame index is available.
6118 SDValue LCandidate = peekThroughBitcasts(N);
6119 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6120 if (FrameIndexSDNode *FINode =
6121 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6122 Op = MachineOperand::CreateFI(FINode->getIndex());
6123 }
6124
6125 if (!Op) {
6126 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6127 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
6128 SplitRegs) {
6129 unsigned Offset = 0;
6130 for (const auto &RegAndSize : SplitRegs) {
6131 // If the expression is already a fragment, the current register
6132 // offset+size might extend beyond the fragment. In this case, only
6133 // the register bits that are inside the fragment are relevant.
6134 int RegFragmentSizeInBits = RegAndSize.second;
6135 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6136 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6137 // The register is entirely outside the expression fragment,
6138 // so is irrelevant for debug info.
6139 if (Offset >= ExprFragmentSizeInBits)
6140 break;
6141 // The register is partially outside the expression fragment, only
6142 // the low bits within the fragment are relevant for debug info.
6143 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6144 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6145 }
6146 }
6147
6148 auto FragmentExpr = DIExpression::createFragmentExpression(
6149 Expr, Offset, RegFragmentSizeInBits);
6150 Offset += RegAndSize.second;
6151 // If a valid fragment expression cannot be created, the variable's
6152 // correct value cannot be determined and so it is set as Undef.
6153 if (!FragmentExpr) {
6155 Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
6156 DAG.AddDbgValue(SDV, false);
6157 continue;
6158 }
6159 MachineInstr *NewMI =
6160 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6161 Kind != FuncArgumentDbgValueKind::Value);
6162 FuncInfo.ArgDbgValues.push_back(NewMI);
6163 }
6164 };
6165
6166 // Check if ValueMap has reg number.
6168 VMI = FuncInfo.ValueMap.find(V);
6169 if (VMI != FuncInfo.ValueMap.end()) {
6170 const auto &TLI = DAG.getTargetLoweringInfo();
6171 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6172 V->getType(), std::nullopt);
6173 if (RFV.occupiesMultipleRegs()) {
6174 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6175 return true;
6176 }
6177
6178 Op = MachineOperand::CreateReg(VMI->second, false);
6179 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6180 } else if (ArgRegsAndSizes.size() > 1) {
6181 // This was split due to the calling convention, and no virtual register
6182 // mapping exists for the value.
6183 splitMultiRegDbgValue(ArgRegsAndSizes);
6184 return true;
6185 }
6186 }
6187
6188 if (!Op)
6189 return false;
6190
6192 "Expected inlined-at fields to agree");
6193 MachineInstr *NewMI = nullptr;
6194
6195 if (Op->isReg())
6196 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6197 else
6198 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6199 Variable, Expr);
6200
6201 // Otherwise, use ArgDbgValues.
6202 FuncInfo.ArgDbgValues.push_back(NewMI);
6203 return true;
6204}
6205
6206/// Return the appropriate SDDbgValue based on N.
6207SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6208 DILocalVariable *Variable,
6209 DIExpression *Expr,
6210 const DebugLoc &dl,
6211 unsigned DbgSDNodeOrder) {
6212 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6213 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6214 // stack slot locations.
6215 //
6216 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6217 // debug values here after optimization:
6218 //
6219 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6220 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6221 //
6222 // Both describe the direct values of their associated variables.
6223 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6224 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6225 }
6226 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6227 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6228}
6229
6230static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6231 switch (Intrinsic) {
6232 case Intrinsic::smul_fix:
6233 return ISD::SMULFIX;
6234 case Intrinsic::umul_fix:
6235 return ISD::UMULFIX;
6236 case Intrinsic::smul_fix_sat:
6237 return ISD::SMULFIXSAT;
6238 case Intrinsic::umul_fix_sat:
6239 return ISD::UMULFIXSAT;
6240 case Intrinsic::sdiv_fix:
6241 return ISD::SDIVFIX;
6242 case Intrinsic::udiv_fix:
6243 return ISD::UDIVFIX;
6244 case Intrinsic::sdiv_fix_sat:
6245 return ISD::SDIVFIXSAT;
6246 case Intrinsic::udiv_fix_sat:
6247 return ISD::UDIVFIXSAT;
6248 default:
6249 llvm_unreachable("Unhandled fixed point intrinsic");
6250 }
6251}
6252
6253void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
6254 const char *FunctionName) {
6255 assert(FunctionName && "FunctionName must not be nullptr");
6257 FunctionName,
6259 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
6260}
6261
6262/// Given a @llvm.call.preallocated.setup, return the corresponding
6263/// preallocated call.
6264static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6265 assert(cast<CallBase>(PreallocatedSetup)
6267 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6268 "expected call_preallocated_setup Value");
6269 for (const auto *U : PreallocatedSetup->users()) {
6270 auto *UseCall = cast<CallBase>(U);
6271 const Function *Fn = UseCall->getCalledFunction();
6272 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6273 return UseCall;
6274 }
6275 }
6276 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6277}
6278
6279/// If DI is a debug value with an EntryValue expression, lower it using the
6280/// corresponding physical register of the associated Argument value
6281/// (guaranteed to exist by the verifier).
6282bool SelectionDAGBuilder::visitEntryValueDbgValue(
6283 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6284 DIExpression *Expr, DebugLoc DbgLoc) {
6285 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6286 return false;
6287
6288 // These properties are guaranteed by the verifier.
6289 const Argument *Arg = cast<Argument>(Values[0]);
6290 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6291
6292 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6293 if (ArgIt == FuncInfo.ValueMap.end()) {
6294 LLVM_DEBUG(
6295 dbgs() << "Dropping dbg.value: expression is entry_value but "
6296 "couldn't find an associated register for the Argument\n");
6297 return true;
6298 }
6299 Register ArgVReg = ArgIt->getSecond();
6300
6301 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6302 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6304 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6305 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6306 return true;
6307 }
6308 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6309 "couldn't find a physical register\n");
6310 return true;
6311}
6312
6313/// Lower the call to the specified intrinsic function.
6314void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6315 unsigned Intrinsic) {
6316 SDLoc sdl = getCurSDLoc();
6317 switch (Intrinsic) {
6318 case Intrinsic::experimental_convergence_anchor:
6319 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6320 break;
6321 case Intrinsic::experimental_convergence_entry:
6322 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6323 break;
6324 case Intrinsic::experimental_convergence_loop: {
6325 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6326 auto *Token = Bundle->Inputs[0].get();
6327 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6328 getValue(Token)));
6329 break;
6330 }
6331 }
6332}
6333
6334void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6335 unsigned IntrinsicID) {
6336 // For now, we're only lowering an 'add' histogram.
6337 // We can add others later, e.g. saturating adds, min/max.
6338 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6339 "Tried to lower unsupported histogram type");
6340 SDLoc sdl = getCurSDLoc();
6341 Value *Ptr = I.getOperand(0);
6342 SDValue Inc = getValue(I.getOperand(1));
6343 SDValue Mask = getValue(I.getOperand(2));
6344
6346 DataLayout TargetDL = DAG.getDataLayout();
6347 EVT VT = Inc.getValueType();
6348 Align Alignment = DAG.getEVTAlign(VT);
6349
6350 const MDNode *Ranges = getRangeMetadata(I);
6351
6352 SDValue Root = DAG.getRoot();
6353 SDValue Base;
6354 SDValue Index;
6355 ISD::MemIndexType IndexType;
6356 SDValue Scale;
6357 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
6358 I.getParent(), VT.getScalarStoreSize());
6359
6360 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6361
6365 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
6366
6367 if (!UniformBase) {
6369 Index = getValue(Ptr);
6370 IndexType = ISD::SIGNED_SCALED;
6371 Scale =
6373 }
6374
6375 EVT IdxVT = Index.getValueType();
6376 EVT EltTy = IdxVT.getVectorElementType();
6377 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
6378 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
6379 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
6380 }
6381
6382 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6383
6384 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6385 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
6386 Ops, MMO, IndexType);
6387
6388 setValue(&I, Histogram);
6389 DAG.setRoot(Histogram);
6390}
6391
6392/// Lower the call to the specified intrinsic function.
6393void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6394 unsigned Intrinsic) {
6396 SDLoc sdl = getCurSDLoc();
6397 DebugLoc dl = getCurDebugLoc();
6398 SDValue Res;
6399
6401 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6402 Flags.copyFMF(*FPOp);
6403
6404 switch (Intrinsic) {
6405 default:
6406 // By default, turn this into a target intrinsic node.
6407 visitTargetIntrinsic(I, Intrinsic);
6408 return;
6409 case Intrinsic::vscale: {
6410 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6411 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6412 return;
6413 }
6414 case Intrinsic::vastart: visitVAStart(I); return;
6415 case Intrinsic::vaend: visitVAEnd(I); return;
6416 case Intrinsic::vacopy: visitVACopy(I); return;
6417 case Intrinsic::returnaddress:
6419 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6420 getValue(I.getArgOperand(0))));
6421 return;
6422 case Intrinsic::addressofreturnaddress:
6423 setValue(&I,
6425 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6426 return;
6427 case Intrinsic::sponentry:
6428 setValue(&I,
6430 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6431 return;
6432 case Intrinsic::frameaddress:
6435 getValue(I.getArgOperand(0))));
6436 return;
6437 case Intrinsic::read_volatile_register:
6438 case Intrinsic::read_register: {
6439 Value *Reg = I.getArgOperand(0);
6440 SDValue Chain = getRoot();
6442 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6443 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6444 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6445 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6446 setValue(&I, Res);
6447 DAG.setRoot(Res.getValue(1));
6448 return;
6449 }
6450 case Intrinsic::write_register: {
6451 Value *Reg = I.getArgOperand(0);
6452 Value *RegValue = I.getArgOperand(1);
6453 SDValue Chain = getRoot();
6455 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6456 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6457 RegName, getValue(RegValue)));
6458 return;
6459 }
6460 case Intrinsic::memcpy: {
6461 const auto &MCI = cast<MemCpyInst>(I);
6462 SDValue Op1 = getValue(I.getArgOperand(0));
6463 SDValue Op2 = getValue(I.getArgOperand(1));
6464 SDValue Op3 = getValue(I.getArgOperand(2));
6465 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
6466 Align DstAlign = MCI.getDestAlign().valueOrOne();
6467 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6468 Align Alignment = std::min(DstAlign, SrcAlign);
6469 bool isVol = MCI.isVolatile();
6470 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6471 // node.
6472 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6473 SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6474 /* AlwaysInline */ false, &I, std::nullopt,
6475 MachinePointerInfo(I.getArgOperand(0)),
6476 MachinePointerInfo(I.getArgOperand(1)),
6477 I.getAAMetadata(), AA);
6478 updateDAGForMaybeTailCall(MC);
6479 return;
6480 }
6481 case Intrinsic::memcpy_inline: {
6482 const auto &MCI = cast<MemCpyInlineInst>(I);
6483 SDValue Dst = getValue(I.getArgOperand(0));
6484 SDValue Src = getValue(I.getArgOperand(1));
6485 SDValue Size = getValue(I.getArgOperand(2));
6486 assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
6487 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6488 Align DstAlign = MCI.getDestAlign().valueOrOne();
6489 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6490 Align Alignment = std::min(DstAlign, SrcAlign);
6491 bool isVol = MCI.isVolatile();
6492 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6493 // node.
6494 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
6495 /* AlwaysInline */ true, &I, std::nullopt,
6496 MachinePointerInfo(I.getArgOperand(0)),
6497 MachinePointerInfo(I.getArgOperand(1)),
6498 I.getAAMetadata(), AA);
6499 updateDAGForMaybeTailCall(MC);
6500 return;
6501 }
6502 case Intrinsic::memset: {
6503 const auto &MSI = cast<MemSetInst>(I);
6504 SDValue Op1 = getValue(I.getArgOperand(0));
6505 SDValue Op2 = getValue(I.getArgOperand(1));
6506 SDValue Op3 = getValue(I.getArgOperand(2));
6507 // @llvm.memset defines 0 and 1 to both mean no alignment.
6508 Align Alignment = MSI.getDestAlign().valueOrOne();
6509 bool isVol = MSI.isVolatile();
6510 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6511 SDValue MS = DAG.getMemset(
6512 Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
6513 &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6514 updateDAGForMaybeTailCall(MS);
6515 return;
6516 }
6517 case Intrinsic::memset_inline: {
6518 const auto &MSII = cast<MemSetInlineInst>(I);
6519 SDValue Dst = getValue(I.getArgOperand(0));
6520 SDValue Value = getValue(I.getArgOperand(1));
6521 SDValue Size = getValue(I.getArgOperand(2));
6522 assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
6523 // @llvm.memset defines 0 and 1 to both mean no alignment.
6524 Align DstAlign = MSII.getDestAlign().valueOrOne();
6525 bool isVol = MSII.isVolatile();
6526 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6527 SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
6528 /* AlwaysInline */ true, &I,
6529 MachinePointerInfo(I.getArgOperand(0)),
6530 I.getAAMetadata());
6531 updateDAGForMaybeTailCall(MC);
6532 return;
6533 }
6534 case Intrinsic::memmove: {
6535 const auto &MMI = cast<MemMoveInst>(I);
6536 SDValue Op1 = getValue(I.getArgOperand(0));
6537 SDValue Op2 = getValue(I.getArgOperand(1));
6538 SDValue Op3 = getValue(I.getArgOperand(2));
6539 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6540 Align DstAlign = MMI.getDestAlign().valueOrOne();
6541 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6542 Align Alignment = std::min(DstAlign, SrcAlign);
6543 bool isVol = MMI.isVolatile();
6544 // FIXME: Support passing different dest/src alignments to the memmove DAG
6545 // node.
6546 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6547 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, &I,
6548 /* OverrideTailCall */ std::nullopt,
6549 MachinePointerInfo(I.getArgOperand(0)),
6550 MachinePointerInfo(I.getArgOperand(1)),
6551 I.getAAMetadata(), AA);
6552 updateDAGForMaybeTailCall(MM);
6553 return;
6554 }
6555 case Intrinsic::memcpy_element_unordered_atomic: {
6556 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
6557 SDValue Dst = getValue(MI.getRawDest());
6558 SDValue Src = getValue(MI.getRawSource());
6559 SDValue Length = getValue(MI.getLength());
6560
6561 Type *LengthTy = MI.getLength()->getType();
6562 unsigned ElemSz = MI.getElementSizeInBytes();
6563 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6564 SDValue MC =
6565 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6566 isTC, MachinePointerInfo(MI.getRawDest()),
6567 MachinePointerInfo(MI.getRawSource()));
6568 updateDAGForMaybeTailCall(MC);
6569 return;
6570 }
6571 case Intrinsic::memmove_element_unordered_atomic: {
6572 auto &MI = cast<AtomicMemMoveInst>(I);
6573 SDValue Dst = getValue(MI.getRawDest());
6574 SDValue Src = getValue(MI.getRawSource());
6575 SDValue Length = getValue(MI.getLength());
6576
6577 Type *LengthTy = MI.getLength()->getType();
6578 unsigned ElemSz = MI.getElementSizeInBytes();
6579 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6580 SDValue MC =
6581 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6582 isTC, MachinePointerInfo(MI.getRawDest()),
6583 MachinePointerInfo(MI.getRawSource()));
6584 updateDAGForMaybeTailCall(MC);
6585 return;
6586 }
6587 case Intrinsic::memset_element_unordered_atomic: {
6588 auto &MI = cast<AtomicMemSetInst>(I);
6589 SDValue Dst = getValue(MI.getRawDest());
6590 SDValue Val = getValue(MI.getValue());
6591 SDValue Length = getValue(MI.getLength());
6592
6593 Type *LengthTy = MI.getLength()->getType();
6594 unsigned ElemSz = MI.getElementSizeInBytes();
6595 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6596 SDValue MC =
6597 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
6598 isTC, MachinePointerInfo(MI.getRawDest()));
6599 updateDAGForMaybeTailCall(MC);
6600 return;
6601 }
6602 case Intrinsic::call_preallocated_setup: {
6603 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6604 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6605 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6606 getRoot(), SrcValue);
6607 setValue(&I, Res);
6608 DAG.setRoot(Res);
6609 return;
6610 }
6611 case Intrinsic::call_preallocated_arg: {
6612 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6613 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6614 SDValue Ops[3];
6615 Ops[0] = getRoot();
6616 Ops[1] = SrcValue;
6617 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6618 MVT::i32); // arg index
6619 SDValue Res = DAG.getNode(
6621 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6622 setValue(&I, Res);
6623 DAG.setRoot(Res.getValue(1));
6624 return;
6625 }
6626 case Intrinsic::dbg_declare: {
6627 const auto &DI = cast<DbgDeclareInst>(I);
6628 // Debug intrinsics are handled separately in assignment tracking mode.
6629 // Some intrinsics are handled right after Argument lowering.
6630 if (AssignmentTrackingEnabled ||
6632 return;
6633 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DI << "\n");
6634 DILocalVariable *Variable = DI.getVariable();
6635 DIExpression *Expression = DI.getExpression();
6637 // Assume dbg.declare can not currently use DIArgList, i.e.
6638 // it is non-variadic.
6639 assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6640 handleDebugDeclare(DI.getVariableLocationOp(0), Variable, Expression,
6641 DI.getDebugLoc());
6642 return;
6643 }
6644 case Intrinsic::dbg_label: {
6645 const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6646 DILabel *Label = DI.getLabel();
6647 assert(Label && "Missing label");
6648
6649 SDDbgLabel *SDV;
6650 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6651 DAG.AddDbgLabel(SDV);
6652 return;
6653 }
6654 case Intrinsic::dbg_assign: {
6655 // Debug intrinsics are handled separately in assignment tracking mode.
6656 if (AssignmentTrackingEnabled)
6657 return;
6658 // If assignment tracking hasn't been enabled then fall through and treat
6659 // the dbg.assign as a dbg.value.
6660 [[fallthrough]];
6661 }
6662 case Intrinsic::dbg_value: {
6663 // Debug intrinsics are handled separately in assignment tracking mode.
6664 if (AssignmentTrackingEnabled)
6665 return;
6666 const DbgValueInst &DI = cast<DbgValueInst>(I);
6667 assert(DI.getVariable() && "Missing variable");
6668
6669 DILocalVariable *Variable = DI.getVariable();
6672
6673 if (DI.isKillLocation()) {
6674 handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
6675 return;
6676 }
6677
6679 if (Values.empty())
6680 return;
6681
6682 bool IsVariadic = DI.hasArgList();
6683 if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
6684 SDNodeOrder, IsVariadic))
6685 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
6686 DI.getDebugLoc(), SDNodeOrder);
6687 return;
6688 }
6689
6690 case Intrinsic::eh_typeid_for: {
6691 // Find the type id for the given typeinfo.
6692 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6693 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6694 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6695 setValue(&I, Res);
6696 return;
6697 }
6698
6699 case Intrinsic::eh_return_i32:
6700 case Intrinsic::eh_return_i64:
6703 MVT::Other,
6705 getValue(I.getArgOperand(0)),
6706 getValue(I.getArgOperand(1))));
6707 return;
6708 case Intrinsic::eh_unwind_init:
6710 return;
6711 case Intrinsic::eh_dwarf_cfa:
6714 getValue(I.getArgOperand(0))));
6715 return;
6716 case Intrinsic::eh_sjlj_callsite: {
6717 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6718 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
6719
6721 return;
6722 }
6723 case Intrinsic::eh_sjlj_functioncontext: {
6724 // Get and store the index of the function context.
6726 AllocaInst *FnCtx =
6727 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6728 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6730 return;
6731 }
6732 case Intrinsic::eh_sjlj_setjmp: {
6733 SDValue Ops[2];
6734 Ops[0] = getRoot();
6735 Ops[1] = getValue(I.getArgOperand(0));
6737 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6738 setValue(&I, Op.getValue(0));
6739 DAG.setRoot(Op.getValue(1));
6740 return;
6741 }
6742 case Intrinsic::eh_sjlj_longjmp:
6743 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6744 getRoot(), getValue(I.getArgOperand(0))));
6745 return;
6746 case Intrinsic::eh_sjlj_setup_dispatch:
6748 getRoot()));
6749 return;
6750 case Intrinsic::masked_gather:
6751 visitMaskedGather(I);
6752 return;
6753 case Intrinsic::masked_load:
6754 visitMaskedLoad(I);
6755 return;
6756 case Intrinsic::masked_scatter:
6757 visitMaskedScatter(I);
6758 return;
6759 case Intrinsic::masked_store:
6760 visitMaskedStore(I);
6761 return;
6762 case Intrinsic::masked_expandload:
6763 visitMaskedLoad(I, true /* IsExpanding */);
6764 return;
6765 case Intrinsic::masked_compressstore:
6766 visitMaskedStore(I, true /* IsCompressing */);
6767 return;
6768 case Intrinsic::powi:
6769 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6770 getValue(I.getArgOperand(1)), DAG));
6771 return;
6772 case Intrinsic::log:
6773 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6774 return;
6775 case Intrinsic::log2:
6776 setValue(&I,
6777 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6778 return;
6779 case Intrinsic::log10:
6780 setValue(&I,
6781 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6782 return;
6783 case Intrinsic::exp:
6784 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6785 return;
6786 case Intrinsic::exp2:
6787 setValue(&I,
6788 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6789 return;
6790 case Intrinsic::pow:
6791 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6792 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6793 return;
6794 case Intrinsic::sqrt:
6795 case Intrinsic::fabs:
6796 case Intrinsic::sin:
6797 case Intrinsic::cos:
6798 case Intrinsic::tan:
6799 case Intrinsic::asin:
6800 case Intrinsic::acos:
6801 case Intrinsic::atan:
6802 case Intrinsic::sinh:
6803 case Intrinsic::cosh:
6804 case Intrinsic::tanh:
6805 case Intrinsic::exp10:
6806 case Intrinsic::floor:
6807 case Intrinsic::ceil:
6808 case Intrinsic::trunc:
6809 case Intrinsic::rint:
6810 case Intrinsic::nearbyint:
6811 case Intrinsic::round:
6812 case Intrinsic::roundeven:
6813 case Intrinsic::canonicalize: {
6814 unsigned Opcode;
6815 // clang-format off
6816 switch (Intrinsic) {
6817 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6818 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6819 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6820 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6821 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6822 case Intrinsic::tan: Opcode = ISD::FTAN; break;
6823 case Intrinsic::asin: Opcode = ISD::FASIN; break;
6824 case Intrinsic::acos: Opcode = ISD::FACOS; break;
6825 case Intrinsic::atan: Opcode = ISD::FATAN; break;
6826 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
6827 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
6828 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
6829 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6830 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6831 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6832 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6833 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6834 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6835 case Intrinsic::round: Opcode = ISD::FROUND; break;
6836 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6837 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6838 }
6839 // clang-format on
6840
6841 setValue(&I, DAG.getNode(Opcode, sdl,
6842 getValue(I.getArgOperand(0)).getValueType(),
6843 getValue(I.getArgOperand(0)), Flags));
6844 return;
6845 }
6846 case Intrinsic::lround:
6847 case Intrinsic::llround:
6848 case Intrinsic::lrint:
6849 case Intrinsic::llrint: {
6850 unsigned Opcode;
6851 // clang-format off
6852 switch (Intrinsic) {
6853 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6854 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6855 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6856 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6857 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6858 }
6859 // clang-format on
6860
6861 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6862 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6863 getValue(I.getArgOperand(0))));
6864 return;
6865 }
6866 case Intrinsic::minnum:
6868 getValue(I.getArgOperand(0)).getValueType(),
6869 getValue(I.getArgOperand(0)),
6870 getValue(I.getArgOperand(1)), Flags));
6871 return;
6872 case Intrinsic::maxnum:
6874 getValue(I.getArgOperand(0)).getValueType(),
6875 getValue(I.getArgOperand(0)),
6876 getValue(I.getArgOperand(1)), Flags));
6877 return;
6878 case Intrinsic::minimum:
6880 getValue(I.getArgOperand(0)).getValueType(),
6881 getValue(I.getArgOperand(0)),
6882 getValue(I.getArgOperand(1)), Flags));
6883 return;
6884 case Intrinsic::maximum:
6886 getValue(I.getArgOperand(0)).getValueType(),
6887 getValue(I.getArgOperand(0)),
6888 getValue(I.getArgOperand(1)), Flags));
6889 return;
6890 case Intrinsic::minimumnum:
6892 getValue(I.getArgOperand(0)).getValueType(),
6893 getValue(I.getArgOperand(0)),
6894 getValue(I.getArgOperand(1)), Flags));
6895 return;
6896 case Intrinsic::maximumnum:
6898 getValue(I.getArgOperand(0)).getValueType(),
6899 getValue(I.getArgOperand(0)),
6900 getValue(I.getArgOperand(1)), Flags));
6901 return;
6902 case Intrinsic::copysign:
6904 getValue(I.getArgOperand(0)).getValueType(),
6905 getValue(I.getArgOperand(0)),
6906 getValue(I.getArgOperand(1)), Flags));
6907 return;
6908 case Intrinsic::ldexp:
6910 getValue(I.getArgOperand(0)).getValueType(),
6911 getValue(I.getArgOperand(0)),
6912 getValue(I.getArgOperand(1)), Flags));
6913 return;
6914 case Intrinsic::frexp: {
6915 SmallVector<EVT, 2> ValueVTs;
6916 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
6917 SDVTList VTs = DAG.getVTList(ValueVTs);
6918 setValue(&I,
6919 DAG.getNode(ISD::FFREXP, sdl, VTs, getValue(I.getArgOperand(0))));
6920 return;
6921 }
6922 case Intrinsic::arithmetic_fence: {
6924 getValue(I.getArgOperand(0)).getValueType(),
6925 getValue(I.getArgOperand(0)), Flags));
6926 return;
6927 }
6928 case Intrinsic::fma:
6930 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6931 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6932 getValue(I.getArgOperand(2)), Flags));
6933 return;
6934#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
6935 case Intrinsic::INTRINSIC:
6936#include "llvm/IR/ConstrainedOps.def"
6937 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6938 return;
6939#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6940#include "llvm/IR/VPIntrinsics.def"
6941 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6942 return;
6943 case Intrinsic::fptrunc_round: {
6944 // Get the last argument, the metadata and convert it to an integer in the
6945 // call
6946 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6947 std::optional<RoundingMode> RoundMode =
6948 convertStrToRoundingMode(cast<MDString>(MD)->getString());
6949
6950 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6951
6952 // Propagate fast-math-flags from IR to node(s).
6954 Flags.copyFMF(*cast<FPMathOperator>(&I));
6955 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6956
6958 Result = DAG.getNode(
6959 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6960 DAG.getTargetConstant((int)*RoundMode, sdl,
6962 setValue(&I, Result);
6963
6964 return;
6965 }
6966 case Intrinsic::fmuladd: {
6967 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6970 setValue(&I, DAG.getNode(ISD::FMA, sdl,
6971 getValue(I.getArgOperand(0)).getValueType(),
6972 getValue(I.getArgOperand(0)),
6973 getValue(I.getArgOperand(1)),
6974 getValue(I.getArgOperand(2)), Flags));
6975 } else {
6976 // TODO: Intrinsic calls should have fast-math-flags.
6978 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6979 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6981 getValue(I.getArgOperand(0)).getValueType(),
6982 Mul, getValue(I.getArgOperand(2)), Flags);
6983 setValue(&I, Add);
6984 }
6985 return;
6986 }
6987 case Intrinsic::convert_to_fp16:
6988 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6989 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6990 getValue(I.getArgOperand(0)),
6991 DAG.getTargetConstant(0, sdl,
6992 MVT::i32))));
6993 return;
6994 case Intrinsic::convert_from_fp16:
6996 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6997 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6998 getValue(I.getArgOperand(0)))));
6999 return;
7000 case Intrinsic::fptosi_sat: {
7001 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7003 getValue(I.getArgOperand(0)),
7005 return;
7006 }
7007 case Intrinsic::fptoui_sat: {
7008 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7010 getValue(I.getArgOperand(0)),
7012 return;
7013 }
7014 case Intrinsic::set_rounding:
7015 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
7016 {getRoot(), getValue(I.getArgOperand(0))});
7017 setValue(&I, Res);
7018 DAG.setRoot(Res.getValue(0));
7019 return;
7020 case Intrinsic::is_fpclass: {
7021 const DataLayout DLayout = DAG.getDataLayout();
7022 EVT DestVT = TLI.getValueType(DLayout, I.getType());
7023 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
7024 FPClassTest Test = static_cast<FPClassTest>(
7025 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
7027 const Function &F = MF.getFunction();
7028 SDValue Op = getValue(I.getArgOperand(0));
7030 Flags.setNoFPExcept(
7031 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7032 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7033 // expansion can use illegal types. Making expansion early allows
7034 // legalizing these types prior to selection.
7035 if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
7036 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
7037 setValue(&I, Result);
7038 return;
7039 }
7040
7041 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
7042 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
7043 setValue(&I, V);
7044 return;
7045 }
7046 case Intrinsic::get_fpenv: {
7047 const DataLayout DLayout = DAG.getDataLayout();
7048 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
7049 Align TempAlign = DAG.getEVTAlign(EnvVT);
7050 SDValue Chain = getRoot();
7051 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7052 // and temporary storage in stack.
7053 if (TLI.isOperationLegalOrCustom(ISD::GET_FPENV, EnvVT)) {
7054 Res = DAG.getNode(
7055 ISD::GET_FPENV, sdl,
7056 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7057 MVT::Other),
7058 Chain);
7059 } else {
7060 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7061 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7062 auto MPI =
7066 TempAlign);
7067 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7068 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
7069 }
7070 setValue(&I, Res);
7071 DAG.setRoot(Res.getValue(1));
7072 return;
7073 }
7074 case Intrinsic::set_fpenv: {
7075 const DataLayout DLayout = DAG.getDataLayout();
7076 SDValue Env = getValue(I.getArgOperand(0));
7077 EVT EnvVT = Env.getValueType();
7078 Align TempAlign = DAG.getEVTAlign(EnvVT);
7079 SDValue Chain = getRoot();
7080 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7081 // environment from memory.
7082 if (TLI.isOperationLegalOrCustom(ISD::SET_FPENV, EnvVT)) {
7083 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
7084 } else {
7085 // Allocate space in stack, copy environment bits into it and use this
7086 // memory in SET_FPENV_MEM.
7087 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7088 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7089 auto MPI =
7091 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7095 TempAlign);
7096 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7097 }
7098 DAG.setRoot(Chain);
7099 return;
7100 }
7101 case Intrinsic::reset_fpenv:
7102 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
7103 return;
7104 case Intrinsic::get_fpmode:
7105 Res = DAG.getNode(
7106 ISD::GET_FPMODE, sdl,
7107 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7108 MVT::Other),
7109 DAG.getRoot());
7110 setValue(&I, Res);
7111 DAG.setRoot(Res.getValue(1));
7112 return;
7113 case Intrinsic::set_fpmode:
7114 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
7115 getValue(I.getArgOperand(0)));
7116 DAG.setRoot(Res);
7117 return;
7118 case Intrinsic::reset_fpmode: {
7119 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
7120 DAG.setRoot(Res);
7121 return;
7122 }
7123 case Intrinsic::pcmarker: {
7124 SDValue Tmp = getValue(I.getArgOperand(0));
7125 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
7126 return;
7127 }
7128 case Intrinsic::readcyclecounter: {
7129 SDValue Op = getRoot();
7131 DAG.getVTList(MVT::i64, MVT::Other), Op);
7132 setValue(&I, Res);
7133 DAG.setRoot(Res.getValue(1));
7134 return;
7135 }
7136 case Intrinsic::readsteadycounter: {
7137 SDValue Op = getRoot();
7139 DAG.getVTList(MVT::i64, MVT::Other), Op);
7140 setValue(&I, Res);
7141 DAG.setRoot(Res.getValue(1));
7142 return;
7143 }
7144 case Intrinsic::bitreverse:
7146 getValue(I.getArgOperand(0)).getValueType(),
7147 getValue(I.getArgOperand(0))));
7148 return;
7149 case Intrinsic::bswap:
7151 getValue(I.getArgOperand(0)).getValueType(),
7152 getValue(I.getArgOperand(0))));
7153 return;
7154 case Intrinsic::cttz: {
7155 SDValue Arg = getValue(I.getArgOperand(0));
7156 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7157 EVT Ty = Arg.getValueType();
7159 sdl, Ty, Arg));
7160 return;
7161 }
7162 case Intrinsic::ctlz: {
7163 SDValue Arg = getValue(I.getArgOperand(0));
7164 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7165 EVT Ty = Arg.getValueType();
7167 sdl, Ty, Arg));
7168 return;
7169 }
7170 case Intrinsic::ctpop: {
7171 SDValue Arg = getValue(I.getArgOperand(0));
7172 EVT Ty = Arg.getValueType();
7173 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7174 return;
7175 }
7176 case Intrinsic::fshl:
7177 case Intrinsic::fshr: {
7178 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7179 SDValue X = getValue(I.getArgOperand(0));
7180 SDValue Y = getValue(I.getArgOperand(1));
7181 SDValue Z = getValue(I.getArgOperand(2));
7182 EVT VT = X.getValueType();
7183
7184 if (X == Y) {
7185 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7186 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7187 } else {
7188 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7189 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7190 }
7191 return;
7192 }
7193 case Intrinsic::sadd_sat: {
7194 SDValue Op1 = getValue(I.getArgOperand(0));
7195 SDValue Op2 = getValue(I.getArgOperand(1));
7196 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7197 return;
7198 }
7199 case Intrinsic::uadd_sat: {
7200 SDValue Op1 = getValue(I.getArgOperand(0));
7201 SDValue Op2 = getValue(I.getArgOperand(1));
7202 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7203 return;
7204 }
7205 case Intrinsic::ssub_sat: {
7206 SDValue Op1 = getValue(I.getArgOperand(0));
7207 SDValue Op2 = getValue(I.getArgOperand(1));
7208 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7209 return;
7210 }
7211 case Intrinsic::usub_sat: {
7212 SDValue Op1 = getValue(I.getArgOperand(0));
7213 SDValue Op2 = getValue(I.getArgOperand(1));
7214 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7215 return;
7216 }
7217 case Intrinsic::sshl_sat: {
7218 SDValue Op1 = getValue(I.getArgOperand(0));
7219 SDValue Op2 = getValue(I.getArgOperand(1));
7220 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7221 return;
7222 }
7223 case Intrinsic::ushl_sat: {
7224 SDValue Op1 = getValue(I.getArgOperand(0));
7225 SDValue Op2 = getValue(I.getArgOperand(1));
7226 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7227 return;
7228 }
7229 case Intrinsic::smul_fix:
7230 case Intrinsic::umul_fix:
7231 case Intrinsic::smul_fix_sat:
7232 case Intrinsic::umul_fix_sat: {
7233 SDValue Op1 = getValue(I.getArgOperand(0));
7234 SDValue Op2 = getValue(I.getArgOperand(1));
7235 SDValue Op3 = getValue(I.getArgOperand(2));
7237 Op1.getValueType(), Op1, Op2, Op3));
7238 return;
7239 }
7240 case Intrinsic::sdiv_fix:
7241 case Intrinsic::udiv_fix:
7242 case Intrinsic::sdiv_fix_sat:
7243 case Intrinsic::udiv_fix_sat: {
7244 SDValue Op1 = getValue(I.getArgOperand(0));
7245 SDValue Op2 = getValue(I.getArgOperand(1));
7246 SDValue Op3 = getValue(I.getArgOperand(2));
7248 Op1, Op2, Op3, DAG, TLI));
7249 return;
7250 }
7251 case Intrinsic::smax: {
7252 SDValue Op1 = getValue(I.getArgOperand(0));
7253 SDValue Op2 = getValue(I.getArgOperand(1));
7254 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7255 return;
7256 }
7257 case Intrinsic::smin: {
7258 SDValue Op1 = getValue(I.getArgOperand(0));
7259 SDValue Op2 = getValue(I.getArgOperand(1));
7260 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7261 return;
7262 }
7263 case Intrinsic::umax: {
7264 SDValue Op1 = getValue(I.getArgOperand(0));
7265 SDValue Op2 = getValue(I.getArgOperand(1));
7266 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7267 return;
7268 }
7269 case Intrinsic::umin: {
7270 SDValue Op1 = getValue(I.getArgOperand(0));
7271 SDValue Op2 = getValue(I.getArgOperand(1));
7272 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7273 return;
7274 }
7275 case Intrinsic::abs: {
7276 // TODO: Preserve "int min is poison" arg in SDAG?
7277 SDValue Op1 = getValue(I.getArgOperand(0));
7278 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
7279 return;
7280 }
7281 case Intrinsic::scmp: {
7282 SDValue Op1 = getValue(I.getArgOperand(0));
7283 SDValue Op2 = getValue(I.getArgOperand(1));
7284 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7285 setValue(&I, DAG.getNode(ISD::SCMP, sdl, DestVT, Op1, Op2));
7286 break;
7287 }
7288 case Intrinsic::ucmp: {
7289 SDValue Op1 = getValue(I.getArgOperand(0));
7290 SDValue Op2 = getValue(I.getArgOperand(1));
7291 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7292 setValue(&I, DAG.getNode(ISD::UCMP, sdl, DestVT, Op1, Op2));
7293 break;
7294 }
7295 case Intrinsic::stacksave: {
7296 SDValue Op = getRoot();
7297 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7298 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
7299 setValue(&I, Res);
7300 DAG.setRoot(Res.getValue(1));
7301 return;
7302 }
7303 case Intrinsic::stackrestore:
7304 Res = getValue(I.getArgOperand(0));
7305 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7306 return;
7307 case Intrinsic::get_dynamic_area_offset: {
7308 SDValue Op = getRoot();
7309 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7310 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7311 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
7312 // target.
7313 if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
7314 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
7315 " intrinsic!");
7317 Op);
7318 DAG.setRoot(Op);
7319 setValue(&I, Res);
7320 return;
7321 }
7322 case Intrinsic::stackguard: {
7324 const Module &M = *MF.getFunction().getParent();
7325 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7326 SDValue Chain = getRoot();
7327 if (TLI.useLoadStackGuardNode()) {
7328 Res = getLoadStackGuard(DAG, sdl, Chain);
7329 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7330 } else {
7331 const Value *Global = TLI.getSDagStackGuard(M);
7333 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7336 }
7337 if (TLI.useStackGuardXorFP())
7338 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
7339 DAG.setRoot(Chain);
7340 setValue(&I, Res);
7341 return;
7342 }
7343 case Intrinsic::stackprotector: {
7344 // Emit code into the DAG to store the stack guard onto the stack.
7346 MachineFrameInfo &MFI = MF.getFrameInfo();
7347 SDValue Src, Chain = getRoot();
7348
7349 if (TLI.useLoadStackGuardNode())
7350 Src = getLoadStackGuard(DAG, sdl, Chain);
7351 else
7352 Src = getValue(I.getArgOperand(0)); // The guard's value.
7353
7354 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7355
7356 int FI = FuncInfo.StaticAllocaMap[Slot];
7357 MFI.setStackProtectorIndex(FI);
7358 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7359
7360 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7361
7362 // Store the stack protector onto the stack.
7363 Res = DAG.getStore(
7364 Chain, sdl, Src, FIN,
7367 setValue(&I, Res);
7368 DAG.setRoot(Res);
7369 return;
7370 }
7371 case Intrinsic::objectsize:
7372 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7373
7374 case Intrinsic::is_constant:
7375 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7376
7377 case Intrinsic::annotation:
7378 case Intrinsic::ptr_annotation:
7379 case Intrinsic::launder_invariant_group:
7380 case Intrinsic::strip_invariant_group:
7381 // Drop the intrinsic, but forward the value
7382 setValue(&I, getValue(I.getOperand(0)));
7383 return;
7384
7385 case Intrinsic::assume:
7386 case Intrinsic::experimental_noalias_scope_decl:
7387 case Intrinsic::var_annotation:
7388 case Intrinsic::sideeffect:
7389 // Discard annotate attributes, noalias scope declarations, assumptions, and
7390 // artificial side-effects.
7391 return;
7392
7393 case Intrinsic::codeview_annotation: {
7394 // Emit a label associated with this metadata.
7396 MCSymbol *Label = MF.getContext().createTempSymbol("annotation", true);
7397 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7398 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7399 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7400 DAG.setRoot(Res);
7401 return;
7402 }
7403
7404 case Intrinsic::init_trampoline: {
7405 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7406
7407 SDValue Ops[6];
7408 Ops[0] = getRoot();
7409 Ops[1] = getValue(I.getArgOperand(0));
7410 Ops[2] = getValue(I.getArgOperand(1));
7411 Ops[3] = getValue(I.getArgOperand(2));
7412 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7413 Ops[5] = DAG.getSrcValue(F);
7414
7415 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7416
7417 DAG.setRoot(Res);
7418 return;
7419 }
7420 case Intrinsic::adjust_trampoline:
7423 getValue(I.getArgOperand(0))));
7424 return;
7425 case Intrinsic::gcroot: {
7427 "only valid in functions with gc specified, enforced by Verifier");
7428 assert(GFI && "implied by previous");
7429 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7430 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7431
7432 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7433 GFI->addStackRoot(FI->getIndex(), TypeMap);
7434 return;
7435 }
7436 case Intrinsic::gcread:
7437 case Intrinsic::gcwrite:
7438 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7439 case Intrinsic::get_rounding:
7440 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7441 setValue(&I, Res);
7442 DAG.setRoot(Res.getValue(1));
7443 return;
7444
7445 case Intrinsic::expect:
7446 // Just replace __builtin_expect(exp, c) with EXP.
7447 setValue(&I, getValue(I.getArgOperand(0)));
7448 return;
7449
7450 case Intrinsic::ubsantrap:
7451 case Intrinsic::debugtrap:
7452 case Intrinsic::trap: {
7453 StringRef TrapFuncName =
7454 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
7455 if (TrapFuncName.empty()) {
7456 switch (Intrinsic) {
7457 case Intrinsic::trap:
7458 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
7459 break;
7460 case Intrinsic::debugtrap:
7461 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
7462 break;
7463 case Intrinsic::ubsantrap:
7465 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
7467 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
7468 MVT::i32)));
7469 break;
7470 default: llvm_unreachable("unknown trap intrinsic");
7471 }
7473 I.hasFnAttr(Attribute::NoMerge));
7474 return;
7475 }
7477 if (Intrinsic == Intrinsic::ubsantrap) {
7479 Args[0].Val = I.getArgOperand(0);
7480 Args[0].Node = getValue(Args[0].Val);
7481 Args[0].Ty = Args[0].Val->getType();
7482 }
7483
7485 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7486 CallingConv::C, I.getType(),
7487 DAG.getExternalSymbol(TrapFuncName.data(),
7489 std::move(Args));
7490 CLI.NoMerge = I.hasFnAttr(Attribute::NoMerge);
7491 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7492 DAG.setRoot(Result.second);
7493 return;
7494 }
7495
7496 case Intrinsic::allow_runtime_check:
7497 case Intrinsic::allow_ubsan_check:
7498 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
7499 return;
7500
7501 case Intrinsic::uadd_with_overflow:
7502 case Intrinsic::sadd_with_overflow:
7503 case Intrinsic::usub_with_overflow:
7504 case Intrinsic::ssub_with_overflow:
7505 case Intrinsic::umul_with_overflow:
7506 case Intrinsic::smul_with_overflow: {
7508 switch (Intrinsic) {
7509 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7510 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7511 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7512 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7513 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7514 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7515 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7516 }
7517 SDValue Op1 = getValue(I.getArgOperand(0));
7518 SDValue Op2 = getValue(I.getArgOperand(1));
7519
7520 EVT ResultVT = Op1.getValueType();
7521 EVT OverflowVT = MVT::i1;
7522 if (ResultVT.isVector())
7523 OverflowVT = EVT::getVectorVT(
7524 *Context, OverflowVT, ResultVT.getVectorElementCount());
7525
7526 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
7527 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
7528 return;
7529 }
7530 case Intrinsic::prefetch: {
7531 SDValue Ops[5];
7532 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7534 Ops[0] = DAG.getRoot();
7535 Ops[1] = getValue(I.getArgOperand(0));
7536 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7537 MVT::i32);
7538 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
7539 MVT::i32);
7540 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
7541 MVT::i32);
7543 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
7544 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
7545 /* align */ std::nullopt, Flags);
7546
7547 // Chain the prefetch in parallel with any pending loads, to stay out of
7548 // the way of later optimizations.
7549 PendingLoads.push_back(Result);
7550 Result = getRoot();
7551 DAG.setRoot(Result);
7552 return;
7553 }
7554 case Intrinsic::lifetime_start:
7555 case Intrinsic::lifetime_end: {
7556 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7557 // Stack coloring is not enabled in O0, discard region information.
7559 return;
7560
7561 const int64_t ObjectSize =
7562 cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
7563 Value *const ObjectPtr = I.getArgOperand(1);
7565 getUnderlyingObjects(ObjectPtr, Allocas);
7566
7567 for (const Value *Alloca : Allocas) {
7568 const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
7569
7570 // Could not find an Alloca.
7571 if (!LifetimeObject)
7572 continue;
7573
7574 // First check that the Alloca is static, otherwise it won't have a
7575 // valid frame index.
7576 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
7577 if (SI == FuncInfo.StaticAllocaMap.end())
7578 return;
7579
7580 const int FrameIndex = SI->second;
7581 int64_t Offset;
7583 ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
7584 Offset = -1; // Cannot determine offset from alloca to lifetime object.
7585 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
7586 Offset);
7587 DAG.setRoot(Res);
7588 }
7589 return;
7590 }
7591 case Intrinsic::pseudoprobe: {
7592 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
7593 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7594 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
7595 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
7596 DAG.setRoot(Res);
7597 return;
7598 }
7599 case Intrinsic::invariant_start:
7600 // Discard region information.
7601 setValue(&I,
7602 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
7603 return;
7604 case Intrinsic::invariant_end:
7605 // Discard region information.
7606 return;
7607 case Intrinsic::clear_cache: {
7608 SDValue InputChain = DAG.getRoot();
7609 SDValue StartVal = getValue(I.getArgOperand(0));
7610 SDValue EndVal = getValue(I.getArgOperand(1));
7611 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),
7612 {InputChain, StartVal, EndVal});
7613 setValue(&I, Res);
7614 DAG.setRoot(Res);
7615 return;
7616 }
7617 case Intrinsic::donothing:
7618 case Intrinsic::seh_try_begin:
7619 case Intrinsic::seh_scope_begin:
7620 case Intrinsic::seh_try_end:
7621 case Intrinsic::seh_scope_end:
7622 // ignore
7623 return;
7624 case Intrinsic::experimental_stackmap:
7625 visitStackmap(I);
7626 return;
7627 case Intrinsic::experimental_patchpoint_void:
7628 case Intrinsic::experimental_patchpoint:
7629 visitPatchpoint(I);
7630 return;
7631 case Intrinsic::experimental_gc_statepoint:
7632 LowerStatepoint(cast<GCStatepointInst>(I));
7633 return;
7634 case Intrinsic::experimental_gc_result:
7635 visitGCResult(cast<GCResultInst>(I));
7636 return;
7637 case Intrinsic::experimental_gc_relocate:
7638 visitGCRelocate(cast<GCRelocateInst>(I));
7639 return;
7640 case Intrinsic::instrprof_cover:
7641 llvm_unreachable("instrprof failed to lower a cover");
7642 case Intrinsic::instrprof_increment:
7643 llvm_unreachable("instrprof failed to lower an increment");
7644 case Intrinsic::instrprof_timestamp:
7645 llvm_unreachable("instrprof failed to lower a timestamp");
7646 case Intrinsic::instrprof_value_profile:
7647 llvm_unreachable("instrprof failed to lower a value profiling call");
7648 case Intrinsic::instrprof_mcdc_parameters:
7649 llvm_unreachable("instrprof failed to lower mcdc parameters");
7650 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7651 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7652 case Intrinsic::localescape: {
7655
7656 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7657 // is the same on all targets.
7658 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7659 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
7660 if (isa<ConstantPointerNull>(Arg))
7661 continue; // Skip null pointers. They represent a hole in index space.
7662 AllocaInst *Slot = cast<AllocaInst>(Arg);
7663 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7664 "can only escape static allocas");
7665 int FI = FuncInfo.StaticAllocaMap[Slot];
7666 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7669 TII->get(TargetOpcode::LOCAL_ESCAPE))
7670 .addSym(FrameAllocSym)
7671 .addFrameIndex(FI);
7672 }
7673
7674 return;
7675 }
7676
7677 case Intrinsic::localrecover: {
7678 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7680
7681 // Get the symbol that defines the frame offset.
7682 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
7683 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
7684 unsigned IdxVal =
7685 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
7686 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7688
7689 Value *FP = I.getArgOperand(1);
7690 SDValue FPVal = getValue(FP);
7691 EVT PtrVT = FPVal.getValueType();
7692
7693 // Create a MCSymbol for the label to avoid any target lowering
7694 // that would make this PC relative.
7695 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
7696 SDValue OffsetVal =
7697 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
7698
7699 // Add the offset to the FP.
7700 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
7701 setValue(&I, Add);
7702
7703 return;
7704 }
7705
7706 case Intrinsic::eh_exceptionpointer:
7707 case Intrinsic::eh_exceptioncode: {
7708 // Get the exception pointer vreg, copy from it, and resize it to fit.
7709 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7710 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7711 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7712 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7713 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7714 if (Intrinsic == Intrinsic::eh_exceptioncode)
7715 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7716 setValue(&I, N);
7717 return;
7718 }
7719 case Intrinsic::xray_customevent: {
7720 // Here we want to make sure that the intrinsic behaves as if it has a
7721 // specific calling convention.
7722 const auto &Triple = DAG.getTarget().getTargetTriple();
7723 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7724 return;
7725
7727
7728 // We want to say that we always want the arguments in registers.
7729 SDValue LogEntryVal = getValue(I.getArgOperand(0));
7730 SDValue StrSizeVal = getValue(I.getArgOperand(1));
7731 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7732 SDValue Chain = getRoot();
7733 Ops.push_back(LogEntryVal);
7734 Ops.push_back(StrSizeVal);
7735 Ops.push_back(Chain);
7736
7737 // We need to enforce the calling convention for the callsite, so that
7738 // argument ordering is enforced correctly, and that register allocation can
7739 // see that some registers may be assumed clobbered and have to preserve
7740 // them across calls to the intrinsic.
7741 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7742 sdl, NodeTys, Ops);
7743 SDValue patchableNode = SDValue(MN, 0);
7744 DAG.setRoot(patchableNode);
7745 setValue(&I, patchableNode);
7746 return;
7747 }
7748 case Intrinsic::xray_typedevent: {
7749 // Here we want to make sure that the intrinsic behaves as if it has a
7750 // specific calling convention.
7751 const auto &Triple = DAG.getTarget().getTargetTriple();
7752 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7753 return;
7754
7756
7757 // We want to say that we always want the arguments in registers.
7758 // It's unclear to me how manipulating the selection DAG here forces callers
7759 // to provide arguments in registers instead of on the stack.
7760 SDValue LogTypeId = getValue(I.getArgOperand(0));
7761 SDValue LogEntryVal = getValue(I.getArgOperand(1));
7762 SDValue StrSizeVal = getValue(I.getArgOperand(2));
7763 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7764 SDValue Chain = getRoot();
7765 Ops.push_back(LogTypeId);
7766 Ops.push_back(LogEntryVal);
7767 Ops.push_back(StrSizeVal);
7768 Ops.push_back(Chain);
7769
7770 // We need to enforce the calling convention for the callsite, so that
7771 // argument ordering is enforced correctly, and that register allocation can
7772 // see that some registers may be assumed clobbered and have to preserve
7773 // them across calls to the intrinsic.
7775 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7776 SDValue patchableNode = SDValue(MN, 0);
7777 DAG.setRoot(patchableNode);
7778 setValue(&I, patchableNode);
7779 return;
7780 }
7781 case Intrinsic::experimental_deoptimize:
7783 return;
7784 case Intrinsic::experimental_stepvector:
7785 visitStepVector(I);
7786 return;
7787 case Intrinsic::vector_reduce_fadd:
7788 case Intrinsic::vector_reduce_fmul:
7789 case Intrinsic::vector_reduce_add:
7790 case Intrinsic::vector_reduce_mul:
7791 case Intrinsic::vector_reduce_and:
7792 case Intrinsic::vector_reduce_or:
7793 case Intrinsic::vector_reduce_xor:
7794 case Intrinsic::vector_reduce_smax:
7795 case Intrinsic::vector_reduce_smin:
7796 case Intrinsic::vector_reduce_umax:
7797 case Intrinsic::vector_reduce_umin:
7798 case Intrinsic::vector_reduce_fmax:
7799 case Intrinsic::vector_reduce_fmin:
7800 case Intrinsic::vector_reduce_fmaximum:
7801 case Intrinsic::vector_reduce_fminimum:
7802 visitVectorReduce(I, Intrinsic);
7803 return;
7804
7805 case Intrinsic::icall_branch_funnel: {
7807 Ops.push_back(getValue(I.getArgOperand(0)));
7808
7809 int64_t Offset;
7810 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7811 I.getArgOperand(1), Offset, DAG.getDataLayout()));
7812 if (!Base)
7814 "llvm.icall.branch.funnel operand must be a GlobalValue");
7815 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7816
7817 struct BranchFunnelTarget {
7818 int64_t Offset;
7820 };
7822
7823 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7824 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7825 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7826 if (ElemBase != Base)
7827 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7828 "to the same GlobalValue");
7829
7830 SDValue Val = getValue(I.getArgOperand(Op + 1));
7831 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7832 if (!GA)
7834 "llvm.icall.branch.funnel operand must be a GlobalValue");
7836 GA->getGlobal(), sdl, Val.getValueType(),
7837 GA->getOffset())});
7838 }
7839 llvm::sort(Targets,
7840 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7841 return T1.Offset < T2.Offset;
7842 });
7843
7844 for (auto &T : Targets) {
7845 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7846 Ops.push_back(T.Target);
7847 }
7848
7849 Ops.push_back(DAG.getRoot()); // Chain
7850 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7851 MVT::Other, Ops),
7852 0);
7853 DAG.setRoot(N);
7854 setValue(&I, N);
7855 HasTailCall = true;
7856 return;
7857 }
7858
7859 case Intrinsic::wasm_landingpad_index:
7860 // Information this intrinsic contained has been transferred to
7861 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7862 // delete it now.
7863 return;
7864
7865 case Intrinsic::aarch64_settag:
7866 case Intrinsic::aarch64_settag_zero: {
7868 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7870 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7871 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7872 ZeroMemory);
7873 DAG.setRoot(Val);
7874 setValue(&I, Val);
7875 return;
7876 }
7877 case Intrinsic::amdgcn_cs_chain: {
7878 assert(I.arg_size() == 5 && "Additional args not supported yet");
7879 assert(cast<ConstantInt>(I.getOperand(4))->isZero() &&
7880 "Non-zero flags not supported yet");
7881
7882 // At this point we don't care if it's amdgpu_cs_chain or
7883 // amdgpu_cs_chain_preserve.
7885
7886 Type *RetTy = I.getType();
7887 assert(RetTy->isVoidTy() && "Should not return");
7888
7889 SDValue Callee = getValue(I.getOperand(0));
7890
7891 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
7892 // We'll also tack the value of the EXEC mask at the end.
7894 Args.reserve(3);
7895
7896 for (unsigned Idx : {2, 3, 1}) {
7898 Arg.Node = getValue(I.getOperand(Idx));
7899 Arg.Ty = I.getOperand(Idx)->getType();
7900 Arg.setAttributes(&I, Idx);
7901 Args.push_back(Arg);
7902 }
7903
7904 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
7905 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
7906 Args[2].IsInReg = true; // EXEC should be inreg
7907
7909 CLI.setDebugLoc(getCurSDLoc())
7910 .setChain(getRoot())
7911 .setCallee(CC, RetTy, Callee, std::move(Args))
7912 .setNoReturn(true)
7913 .setTailCall(true)
7914 .setConvergent(I.isConvergent());
7915 CLI.CB = &I;
7916 std::pair<SDValue, SDValue> Result =
7917 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
7918 (void)Result;
7919 assert(!Result.first.getNode() && !Result.second.getNode() &&
7920 "Should've lowered as tail call");
7921
7922 HasTailCall = true;
7923 return;
7924 }
7925 case Intrinsic::ptrmask: {
7926 SDValue Ptr = getValue(I.getOperand(0));
7927 SDValue Mask = getValue(I.getOperand(1));
7928
7929 // On arm64_32, pointers are 32 bits when stored in memory, but
7930 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
7931 // match the index type, but the pointer is 64 bits, so the the mask must be
7932 // zero-extended up to 64 bits to match the pointer.
7933 EVT PtrVT =
7934 TLI.getValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
7935 EVT MemVT =
7936 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
7937 assert(PtrVT == Ptr.getValueType());
7938 assert(MemVT == Mask.getValueType());
7939 if (MemVT != PtrVT)
7940 Mask = DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);
7941
7942 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
7943 return;
7944 }
7945 case Intrinsic::threadlocal_address: {
7946 setValue(&I, getValue(I.getOperand(0)));
7947 return;
7948 }
7949 case Intrinsic::get_active_lane_mask: {
7950 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7951 SDValue Index = getValue(I.getOperand(0));
7952 EVT ElementVT = Index.getValueType();
7953
7954 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7955 visitTargetIntrinsic(I, Intrinsic);
7956 return;
7957 }
7958
7959 SDValue TripCount = getValue(I.getOperand(1));
7960 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
7961 CCVT.getVectorElementCount());
7962
7963 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
7964 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
7965 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7966 SDValue VectorInduction = DAG.getNode(
7967 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7968 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7969 VectorTripCount, ISD::CondCode::SETULT);
7970 setValue(&I, SetCC);
7971 return;
7972 }
7973 case Intrinsic::experimental_get_vector_length: {
7974 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
7975 "Expected positive VF");
7976 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
7977 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
7978
7979 SDValue Count = getValue(I.getOperand(0));
7980 EVT CountVT = Count.getValueType();
7981
7982 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
7983 visitTargetIntrinsic(I, Intrinsic);
7984 return;
7985 }
7986
7987 // Expand to a umin between the trip count and the maximum elements the type
7988 // can hold.
7989 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7990
7991 // Extend the trip count to at least the result VT.
7992 if (CountVT.bitsLT(VT)) {
7993 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
7994 CountVT = VT;
7995 }
7996
7997 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
7998 ElementCount::get(VF, IsScalable));
7999
8000 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
8001 // Clip to the result type if needed.
8002 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
8003
8004 setValue(&I, Trunc);
8005 return;
8006 }
8007 case Intrinsic::experimental_vector_partial_reduce_add: {
8008 SDValue OpNode = getValue(I.getOperand(1));
8009 EVT ReducedTy = EVT::getEVT(I.getType());
8010 EVT FullTy = OpNode.getValueType();
8011
8012 unsigned Stride = ReducedTy.getVectorMinNumElements();
8013 unsigned ScaleFactor = FullTy.getVectorMinNumElements() / Stride;
8014
8015 // Collect all of the subvectors
8016 std::deque<SDValue> Subvectors;
8017 Subvectors.push_back(getValue(I.getOperand(0)));
8018 for (unsigned i = 0; i < ScaleFactor; i++) {
8019 auto SourceIndex = DAG.getVectorIdxConstant(i * Stride, sdl);
8020 Subvectors.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ReducedTy,
8021 {OpNode, SourceIndex}));
8022 }
8023
8024 // Flatten the subvector tree
8025 while (Subvectors.size() > 1) {
8026 Subvectors.push_back(DAG.getNode(ISD::ADD, sdl, ReducedTy,
8027 {Subvectors[0], Subvectors[1]}));
8028 Subvectors.pop_front();
8029 Subvectors.pop_front();
8030 }
8031
8032 assert(Subvectors.size() == 1 &&
8033 "There should only be one subvector after tree flattening");
8034
8035 setValue(&I, Subvectors[0]);
8036 return;
8037 }
8038 case Intrinsic::experimental_cttz_elts: {
8039 auto DL = getCurSDLoc();
8040 SDValue Op = getValue(I.getOperand(0));
8041 EVT OpVT = Op.getValueType();
8042
8043 if (!TLI.shouldExpandCttzElements(OpVT)) {
8044 visitTargetIntrinsic(I, Intrinsic);
8045 return;
8046 }
8047
8048 if (OpVT.getScalarType() != MVT::i1) {
8049 // Compare the input vector elements to zero & use to count trailing zeros
8050 SDValue AllZero = DAG.getConstant(0, DL, OpVT);
8051 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
8052 OpVT.getVectorElementCount());
8053 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
8054 }
8055
8056 // If the zero-is-poison flag is set, we can assume the upper limit
8057 // of the result is VF-1.
8058 bool ZeroIsPoison =
8059 !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
8060 ConstantRange VScaleRange(1, true); // Dummy value.
8061 if (isa<ScalableVectorType>(I.getOperand(0)->getType()))
8062 VScaleRange = getVScaleRange(I.getCaller(), 64);
8063 unsigned EltWidth = TLI.getBitWidthForCttzElements(
8064 I.getType(), OpVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
8065
8066 MVT NewEltTy = MVT::getIntegerVT(EltWidth);
8067
8068 // Create the new vector type & get the vector length
8069 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,
8070 OpVT.getVectorElementCount());
8071
8072 SDValue VL =
8073 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());
8074
8075 SDValue StepVec = DAG.getStepVector(DL, NewVT);
8076 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
8077 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
8079 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
8081 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
8082
8083 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
8085
8086 setValue(&I, Ret);
8087 return;
8088 }
8089 case Intrinsic::vector_insert: {
8090 SDValue Vec = getValue(I.getOperand(0));
8091 SDValue SubVec = getValue(I.getOperand(1));
8092 SDValue Index = getValue(I.getOperand(2));
8093
8094 // The intrinsic's index type is i64, but the SDNode requires an index type
8095 // suitable for the target. Convert the index as required.
8096 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8097 if (Index.getValueType() != VectorIdxTy)
8098 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8099
8100 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8101 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
8102 Index));
8103 return;
8104 }
8105 case Intrinsic::vector_extract: {
8106 SDValue Vec = getValue(I.getOperand(0));
8107 SDValue Index = getValue(I.getOperand(1));
8108 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8109
8110 // The intrinsic's index type is i64, but the SDNode requires an index type
8111 // suitable for the target. Convert the index as required.
8112 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8113 if (Index.getValueType() != VectorIdxTy)
8114 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8115
8116 setValue(&I,
8117 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
8118 return;
8119 }
8120 case Intrinsic::vector_reverse:
8121 visitVectorReverse(I);
8122 return;
8123 case Intrinsic::vector_splice:
8124 visitVectorSplice(I);
8125 return;
8126 case Intrinsic::callbr_landingpad:
8127 visitCallBrLandingPad(I);
8128 return;
8129 case Intrinsic::vector_interleave2:
8130 visitVectorInterleave(I);
8131 return;
8132 case Intrinsic::vector_deinterleave2:
8133 visitVectorDeinterleave(I);
8134 return;
8135 case Intrinsic::experimental_vector_compress:
8137 getValue(I.getArgOperand(0)).getValueType(),
8138 getValue(I.getArgOperand(0)),
8139 getValue(I.getArgOperand(1)),
8140 getValue(I.getArgOperand(2)), Flags));
8141 return;
8142 case Intrinsic::experimental_convergence_anchor:
8143 case Intrinsic::experimental_convergence_entry:
8144 case Intrinsic::experimental_convergence_loop:
8145 visitConvergenceControl(I, Intrinsic);
8146 return;
8147 case Intrinsic::experimental_vector_histogram_add: {
8148 visitVectorHistogram(I, Intrinsic);
8149 return;
8150 }
8151 }
8152}
8153
8154void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8155 const ConstrainedFPIntrinsic &FPI) {
8156 SDLoc sdl = getCurSDLoc();
8157
8158 // We do not need to serialize constrained FP intrinsics against
8159 // each other or against (nonvolatile) loads, so they can be
8160 // chained like loads.
8161 SDValue Chain = DAG.getRoot();
8163 Opers.push_back(Chain);
8164 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8165 Opers.push_back(getValue(FPI.getArgOperand(I)));
8166
8167 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
8168 assert(Result.getNode()->getNumValues() == 2);
8169
8170 // Push node to the appropriate list so that future instructions can be
8171 // chained up correctly.
8172 SDValue OutChain = Result.getValue(1);
8173 switch (EB) {
8175 // The only reason why ebIgnore nodes still need to be chained is that
8176 // they might depend on the current rounding mode, and therefore must
8177 // not be moved across instruction that may change that mode.
8178 [[fallthrough]];
8180 // These must not be moved across calls or instructions that may change
8181 // floating-point exception masks.
8182 PendingConstrainedFP.push_back(OutChain);
8183 break;
8185 // These must not be moved across calls or instructions that may change
8186 // floating-point exception masks or read floating-point exception flags.
8187 // In addition, they cannot be optimized out even if unused.
8188 PendingConstrainedFPStrict.push_back(OutChain);
8189 break;
8190 }
8191 };
8192
8194 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
8195 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
8197
8200 Flags.setNoFPExcept(true);
8201
8202 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
8203 Flags.copyFMF(*FPOp);
8204
8205 unsigned Opcode;
8206 switch (FPI.getIntrinsicID()) {
8207 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8208#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8209 case Intrinsic::INTRINSIC: \
8210 Opcode = ISD::STRICT_##DAGN; \
8211 break;
8212#include "llvm/IR/ConstrainedOps.def"
8213 case Intrinsic::experimental_constrained_fmuladd: {
8214 Opcode = ISD::STRICT_FMA;
8215 // Break fmuladd into fmul and fadd.
8218 Opers.pop_back();
8219 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8220 pushOutChain(Mul, EB);
8221 Opcode = ISD::STRICT_FADD;
8222 Opers.clear();
8223 Opers.push_back(Mul.getValue(1));
8224 Opers.push_back(Mul.getValue(0));
8225 Opers.push_back(getValue(FPI.getArgOperand(2)));
8226 }
8227 break;
8228 }
8229 }
8230
8231 // A few strict DAG nodes carry additional operands that are not
8232 // set up by the default code above.
8233 switch (Opcode) {
8234 default: break;
8236 Opers.push_back(
8238 break;
8239 case ISD::STRICT_FSETCC:
8240 case ISD::STRICT_FSETCCS: {
8241 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8242 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8243 if (TM.Options.NoNaNsFPMath)
8244 Condition = getFCmpCodeWithoutNaN(Condition);
8245 Opers.push_back(DAG.getCondCode(Condition));
8246 break;
8247 }
8248 }
8249
8250 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8251 pushOutChain(Result, EB);
8252
8253 SDValue FPResult = Result.getValue(0);
8254 setValue(&FPI, FPResult);
8255}
8256
8257static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8258 std::optional<unsigned> ResOPC;
8259 switch (VPIntrin.getIntrinsicID()) {
8260 case Intrinsic::vp_ctlz: {
8261 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8262 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8263 break;
8264 }
8265 case Intrinsic::vp_cttz: {
8266 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8267 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8268 break;
8269 }
8270 case Intrinsic::vp_cttz_elts: {
8271 bool IsZeroPoison = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8272 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_UNDEF : ISD::VP_CTTZ_ELTS;
8273 break;
8274 }
8275#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8276 case Intrinsic::VPID: \
8277 ResOPC = ISD::VPSD; \
8278 break;
8279#include "llvm/IR/VPIntrinsics.def"
8280 }
8281
8282 if (!ResOPC)
8284 "Inconsistency: no SDNode available for this VPIntrinsic!");
8285
8286 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8287 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8288 if (VPIntrin.getFastMathFlags().allowReassoc())
8289 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8290 : ISD::VP_REDUCE_FMUL;
8291 }
8292
8293 return *ResOPC;
8294}
8295
8296void SelectionDAGBuilder::visitVPLoad(
8297 const VPIntrinsic &VPIntrin, EVT VT,
8298 const SmallVectorImpl<SDValue> &OpValues) {
8299 SDLoc DL = getCurSDLoc();
8300 Value *PtrOperand = VPIntrin.getArgOperand(0);
8301 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8302 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8303 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8304 SDValue LD;
8305 // Do not serialize variable-length loads of constant memory with
8306 // anything.
8307 if (!Alignment)
8308 Alignment = DAG.getEVTAlign(VT);
8309 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8310 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8311 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8314 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8315 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
8316 MMO, false /*IsExpanding */);
8317 if (AddToChain)
8318 PendingLoads.push_back(LD.getValue(1));
8319 setValue(&VPIntrin, LD);
8320}
8321
8322void SelectionDAGBuilder::visitVPGather(
8323 const VPIntrinsic &VPIntrin, EVT VT,
8324 const SmallVectorImpl<SDValue> &OpValues) {
8325 SDLoc DL = getCurSDLoc();
8327 Value *PtrOperand = VPIntrin.getArgOperand(0);
8328 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8329 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8330 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8331 SDValue LD;
8332 if (!Alignment)
8333 Alignment = DAG.getEVTAlign(VT.getScalarType());
8334 unsigned AS =
8335 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8338 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8339 SDValue Base, Index, Scale;
8340 ISD::MemIndexType IndexType;
8341 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8342 this, VPIntrin.getParent(),
8343 VT.getScalarStoreSize());
8344 if (!UniformBase) {
8346 Index = getValue(PtrOperand);
8347 IndexType = ISD::SIGNED_SCALED;
8349 }
8350 EVT IdxVT = Index.getValueType();
8351 EVT EltTy = IdxVT.getVectorElementType();
8352 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8353 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8354 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8355 }
8356 LD = DAG.getGatherVP(
8357 DAG.getVTList(VT, MVT::Other), VT, DL,
8358 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8359 IndexType);
8360 PendingLoads.push_back(LD.getValue(1));
8361 setValue(&VPIntrin, LD);
8362}
8363
8364void SelectionDAGBuilder::visitVPStore(
8365 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8366 SDLoc DL = getCurSDLoc();
8367 Value *PtrOperand = VPIntrin.getArgOperand(1);
8368 EVT VT = OpValues[0].getValueType();
8369 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8370 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8371 SDValue ST;
8372 if (!Alignment)
8373 Alignment = DAG.getEVTAlign(VT);
8374 SDValue Ptr = OpValues[1];
8375 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
8378 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8379 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
8380 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
8381 /* IsTruncating */ false, /*IsCompressing*/ false);
8382 DAG.setRoot(ST);
8383 setValue(&VPIntrin, ST);
8384}
8385
8386void SelectionDAGBuilder::visitVPScatter(
8387 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8388 SDLoc DL = getCurSDLoc();
8390 Value *PtrOperand = VPIntrin.getArgOperand(1);
8391 EVT VT = OpValues[0].getValueType();
8392 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8393 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8394 SDValue ST;
8395 if (!Alignment)
8396 Alignment = DAG.getEVTAlign(VT.getScalarType());
8397 unsigned AS =
8398 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8401 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8402 SDValue Base, Index, Scale;
8403 ISD::MemIndexType IndexType;
8404 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8405 this, VPIntrin.getParent(),
8406 VT.getScalarStoreSize());
8407 if (!UniformBase) {
8409 Index = getValue(PtrOperand);
8410 IndexType = ISD::SIGNED_SCALED;
8411 Scale =
8413 }
8414 EVT IdxVT = Index.getValueType();
8415 EVT EltTy = IdxVT.getVectorElementType();
8416 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8417 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8418 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8419 }
8420 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
8421 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8422 OpValues[2], OpValues[3]},
8423 MMO, IndexType);
8424 DAG.setRoot(ST);
8425 setValue(&VPIntrin, ST);
8426}
8427
8428void SelectionDAGBuilder::visitVPStridedLoad(
8429 const VPIntrinsic &VPIntrin, EVT VT,
8430 const SmallVectorImpl<SDValue> &OpValues) {
8431 SDLoc DL = getCurSDLoc();
8432 Value *PtrOperand = VPIntrin.getArgOperand(0);
8433 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8434 if (!Alignment)
8435 Alignment = DAG.getEVTAlign(VT.getScalarType());
8436 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8437 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8438 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8439 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8440 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8441 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8444 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8445
8446 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
8447 OpValues[2], OpValues[3], MMO,
8448 false /*IsExpanding*/);
8449
8450 if (AddToChain)
8451 PendingLoads.push_back(LD.getValue(1));
8452 setValue(&VPIntrin, LD);
8453}
8454
8455void SelectionDAGBuilder::visitVPStridedStore(
8456 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8457 SDLoc DL = getCurSDLoc();
8458 Value *PtrOperand = VPIntrin.getArgOperand(1);
8459 EVT VT = OpValues[0].getValueType();
8460 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8461 if (!Alignment)
8462 Alignment = DAG.getEVTAlign(VT.getScalarType());
8463 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8464 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8467 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8468
8470 getMemoryRoot(), DL, OpValues[0], OpValues[1],
8471 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
8472 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
8473 /*IsCompressing*/ false);
8474
8475 DAG.setRoot(ST);
8476 setValue(&VPIntrin, ST);
8477}
8478
8479void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8481 SDLoc DL = getCurSDLoc();
8482
8483 ISD::CondCode Condition;
8485 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
8486 if (IsFP) {
8487 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
8488 // flags, but calls that don't return floating-point types can't be
8489 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
8490 Condition = getFCmpCondCode(CondCode);
8491 if (TM.Options.NoNaNsFPMath)
8492 Condition = getFCmpCodeWithoutNaN(Condition);
8493 } else {
8494 Condition = getICmpCondCode(CondCode);
8495 }
8496
8497 SDValue Op1 = getValue(VPIntrin.getOperand(0));
8498 SDValue Op2 = getValue(VPIntrin.getOperand(1));
8499 // #2 is the condition code
8500 SDValue MaskOp = getValue(VPIntrin.getOperand(3));
8501 SDValue EVL = getValue(VPIntrin.getOperand(4));
8502 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8503 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8504 "Unexpected target EVL type");
8505 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
8506
8508 VPIntrin.getType());
8509 setValue(&VPIntrin,
8510 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
8511}
8512
8513void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8514 const VPIntrinsic &VPIntrin) {
8515 SDLoc DL = getCurSDLoc();
8516 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8517
8518 auto IID = VPIntrin.getIntrinsicID();
8519
8520 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
8521 return visitVPCmp(*CmpI);
8522
8523 SmallVector<EVT, 4> ValueVTs;
8525 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
8526 SDVTList VTs = DAG.getVTList(ValueVTs);
8527
8528 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
8529
8530 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8531 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8532 "Unexpected target EVL type");
8533
8534 // Request operands.
8535 SmallVector<SDValue, 7> OpValues;
8536 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8537 auto Op = getValue(VPIntrin.getArgOperand(I));
8538 if (I == EVLParamPos)
8539 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
8540 OpValues.push_back(Op);
8541 }
8542
8543 switch (Opcode) {
8544 default: {
8545 SDNodeFlags SDFlags;
8546 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8547 SDFlags.copyFMF(*FPMO);
8548 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
8549 setValue(&VPIntrin, Result);
8550 break;
8551 }
8552 case ISD::VP_LOAD:
8553 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
8554 break;
8555 case ISD::VP_GATHER:
8556 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
8557 break;
8558 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8559 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
8560 break;
8561 case ISD::VP_STORE:
8562 visitVPStore(VPIntrin, OpValues);
8563 break;
8564 case ISD::VP_SCATTER:
8565 visitVPScatter(VPIntrin, OpValues);
8566 break;
8567 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8568 visitVPStridedStore(VPIntrin, OpValues);
8569 break;
8570 case ISD::VP_FMULADD: {
8571 assert(OpValues.size() == 5 && "Unexpected number of operands");
8572 SDNodeFlags SDFlags;
8573 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8574 SDFlags.copyFMF(*FPMO);
8577 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
8578 } else {
8580 ISD::VP_FMUL, DL, VTs,
8581 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
8582 SDValue Add =
8583 DAG.getNode(ISD::VP_FADD, DL, VTs,
8584 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
8585 setValue(&VPIntrin, Add);
8586 }
8587 break;
8588 }
8589 case ISD::VP_IS_FPCLASS: {
8590 const DataLayout DLayout = DAG.getDataLayout();
8591 EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType());
8592 auto Constant = OpValues[1]->getAsZExtVal();
8594 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT,
8595 {OpValues[0], Check, OpValues[2], OpValues[3]});
8596 setValue(&VPIntrin, V);
8597 return;
8598 }
8599 case ISD::VP_INTTOPTR: {
8600 SDValue N = OpValues[0];
8601 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
8602 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
8603 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8604 OpValues[2]);
8605 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8606 OpValues[2]);
8607 setValue(&VPIntrin, N);
8608 break;
8609 }
8610 case ISD::VP_PTRTOINT: {
8611 SDValue N = OpValues[0];
8613 VPIntrin.getType());
8614 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
8615 VPIntrin.getOperand(0)->getType());
8616 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8617 OpValues[2]);
8618 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8619 OpValues[2]);
8620 setValue(&VPIntrin, N);
8621 break;
8622 }
8623 case ISD::VP_ABS:
8624 case ISD::VP_CTLZ:
8625 case ISD::VP_CTLZ_ZERO_UNDEF:
8626 case ISD::VP_CTTZ:
8627 case ISD::VP_CTTZ_ZERO_UNDEF:
8628 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
8629 case ISD::VP_CTTZ_ELTS: {
8630 SDValue Result =
8631 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
8632 setValue(&VPIntrin, Result);
8633 break;
8634 }
8635 }
8636}
8637
8638SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
8639 const BasicBlock *EHPadBB,
8640 MCSymbol *&BeginLabel) {
8642
8643 // Insert a label before the invoke call to mark the try range. This can be
8644 // used to detect deletion of the invoke via the MachineModuleInfo.
8645 BeginLabel = MF.getContext().createTempSymbol();
8646
8647 // For SjLj, keep track of which landing pads go with which invokes
8648 // so as to maintain the ordering of pads in the LSDA.
8649 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
8650 if (CallSiteIndex) {
8651 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
8652 LPadToCallSiteMap[FuncInfo.getMBB(EHPadBB)].push_back(CallSiteIndex);
8653
8654 // Now that the call site is handled, stop tracking it.
8656 }
8657
8658 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
8659}
8660
8661SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
8662 const BasicBlock *EHPadBB,
8663 MCSymbol *BeginLabel) {
8664 assert(BeginLabel && "BeginLabel should've been set");
8665
8667
8668 // Insert a label at the end of the invoke call to mark the try range. This
8669 // can be used to detect deletion of the invoke via the MachineModuleInfo.
8670 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
8671 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
8672
8673 // Inform MachineModuleInfo of range.
8675 // There is a platform (e.g. wasm) that uses funclet style IR but does not
8676 // actually use outlined funclets and their LSDA info style.
8677 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
8678 assert(II && "II should've been set");
8679 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
8680 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
8681 } else if (!isScopedEHPersonality(Pers)) {
8682 assert(EHPadBB);
8683 MF.addInvoke(FuncInfo.getMBB(EHPadBB), BeginLabel, EndLabel);
8684 }
8685
8686 return Chain;
8687}
8688
8689std::pair<SDValue, SDValue>
8691 const BasicBlock *EHPadBB) {
8692 MCSymbol *BeginLabel = nullptr;
8693
8694 if (EHPadBB) {
8695 // Both PendingLoads and PendingExports must be flushed here;
8696 // this call might not return.
8697 (void)getRoot();
8698 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
8699 CLI.setChain(getRoot());
8700 }
8701
8703 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8704
8705 assert((CLI.IsTailCall || Result.second.getNode()) &&
8706 "Non-null chain expected with non-tail call!");
8707 assert((Result.second.getNode() || !Result.first.getNode()) &&
8708 "Null value expected with tail call!");
8709
8710 if (!Result.second.getNode()) {
8711 // As a special case, a null chain means that a tail call has been emitted
8712 // and the DAG root is already updated.
8713 HasTailCall = true;
8714
8715 // Since there's no actual continuation from this block, nothing can be
8716 // relying on us setting vregs for them.
8717 PendingExports.clear();
8718 } else {
8719 DAG.setRoot(Result.second);
8720 }
8721
8722 if (EHPadBB) {
8723 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
8724 BeginLabel));
8725 Result.second = getRoot();
8726 }
8727
8728 return Result;
8729}
8730
8732 bool isTailCall, bool isMustTailCall,
8733 const BasicBlock *EHPadBB,
8734 const TargetLowering::PtrAuthInfo *PAI) {
8735 auto &DL = DAG.getDataLayout();
8736 FunctionType *FTy = CB.getFunctionType();
8737 Type *RetTy = CB.getType();
8738
8740 Args.reserve(CB.arg_size());
8741
8742 const Value *SwiftErrorVal = nullptr;
8744
8745 if (isTailCall) {
8746 // Avoid emitting tail calls in functions with the disable-tail-calls
8747 // attribute.
8748 auto *Caller = CB.getParent()->getParent();
8749 if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
8750 "true" && !isMustTailCall)
8751 isTailCall = false;
8752
8753 // We can't tail call inside a function with a swifterror argument. Lowering
8754 // does not support this yet. It would have to move into the swifterror
8755 // register before the call.
8756 if (TLI.supportSwiftError() &&
8757 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8758 isTailCall = false;
8759 }
8760
8761 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
8763 const Value *V = *I;
8764
8765 // Skip empty types
8766 if (V->getType()->isEmptyTy())
8767 continue;
8768
8769 SDValue ArgNode = getValue(V);
8770 Entry.Node = ArgNode; Entry.Ty = V->getType();
8771
8772 Entry.setAttributes(&CB, I - CB.arg_begin());
8773
8774 // Use swifterror virtual register as input to the call.
8775 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
8776 SwiftErrorVal = V;
8777 // We find the virtual register for the actual swifterror argument.
8778 // Instead of using the Value, we use the virtual register instead.
8779 Entry.Node =
8781 EVT(TLI.getPointerTy(DL)));
8782 }
8783
8784 Args.push_back(Entry);
8785
8786 // If we have an explicit sret argument that is an Instruction, (i.e., it
8787 // might point to function-local memory), we can't meaningfully tail-call.
8788 if (Entry.IsSRet && isa<Instruction>(V))
8789 isTailCall = false;
8790 }
8791
8792 // If call site has a cfguardtarget operand bundle, create and add an
8793 // additional ArgListEntry.
8794 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
8796 Value *V = Bundle->Inputs[0];
8797 SDValue ArgNode = getValue(V);
8798 Entry.Node = ArgNode;
8799 Entry.Ty = V->getType();
8800 Entry.IsCFGuardTarget = true;
8801 Args.push_back(Entry);
8802 }
8803
8804 // Check if target-independent constraints permit a tail call here.
8805 // Target-dependent constraints are checked within TLI->LowerCallTo.
8806 if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
8807 isTailCall = false;
8808
8809 // Disable tail calls if there is an swifterror argument. Targets have not
8810 // been updated to support tail calls.
8811 if (TLI.supportSwiftError() && SwiftErrorVal)
8812 isTailCall = false;
8813
8814 ConstantInt *CFIType = nullptr;
8815 if (CB.isIndirectCall()) {
8816 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
8817 if (!TLI.supportKCFIBundles())
8819 "Target doesn't support calls with kcfi operand bundles.");
8820 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8821 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8822 }
8823 }
8824
8825 SDValue ConvControlToken;
8826 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8827 auto *Token = Bundle->Inputs[0].get();
8828 ConvControlToken = getValue(Token);
8829 }
8830
8833 .setChain(getRoot())
8834 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
8835 .setTailCall(isTailCall)
8839 .setCFIType(CFIType)
8840 .setConvergenceControlToken(ConvControlToken);
8841
8842 // Set the pointer authentication info if we have it.
8843 if (PAI) {
8844 if (!TLI.supportPtrAuthBundles())
8846 "This target doesn't support calls with ptrauth operand bundles.");
8847 CLI.setPtrAuth(*PAI);
8848 }
8849
8850 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8851
8852 if (Result.first.getNode()) {
8853 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
8854 setValue(&CB, Result.first);
8855 }
8856
8857 // The last element of CLI.InVals has the SDValue for swifterror return.
8858 // Here we copy it to a virtual register and update SwiftErrorMap for
8859 // book-keeping.
8860 if (SwiftErrorVal && TLI.supportSwiftError()) {
8861 // Get the last element of InVals.
8862 SDValue Src = CLI.InVals.back();
8863 Register VReg =
8864 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
8865 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
8866 DAG.setRoot(CopyNode);
8867 }
8868}
8869
8870static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
8871 SelectionDAGBuilder &Builder) {
8872 // Check to see if this load can be trivially constant folded, e.g. if the
8873 // input is from a string literal.
8874 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
8875 // Cast pointer to the type we really want to load.
8876 Type *LoadTy =
8877 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
8878 if (LoadVT.isVector())
8879 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
8880
8881 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
8882 PointerType::getUnqual(LoadTy));
8883
8884 if (const Constant *LoadCst =
8885 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
8886 LoadTy, Builder.DAG.getDataLayout()))
8887 return Builder.getValue(LoadCst);
8888 }
8889
8890 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
8891 // still constant memory, the input chain can be the entry node.
8892 SDValue Root;
8893 bool ConstantMemory = false;
8894
8895 // Do not serialize (non-volatile) loads of constant memory with anything.
8896 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
8897 Root = Builder.DAG.getEntryNode();
8898 ConstantMemory = true;
8899 } else {
8900 // Do not serialize non-volatile loads against each other.
8901 Root = Builder.DAG.getRoot();
8902 }
8903
8904 SDValue Ptr = Builder.getValue(PtrVal);
8905 SDValue LoadVal =
8906 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
8907 MachinePointerInfo(PtrVal), Align(1));
8908
8909 if (!ConstantMemory)
8910 Builder.PendingLoads.push_back(LoadVal.getValue(1));
8911 return LoadVal;
8912}
8913
8914/// Record the value for an instruction that produces an integer result,
8915/// converting the type where necessary.
8916void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
8917 SDValue Value,
8918 bool IsSigned) {
8920 I.getType(), true);
8921 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
8922 setValue(&I, Value);
8923}
8924
8925/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
8926/// true and lower it. Otherwise return false, and it will be lowered like a
8927/// normal call.
8928/// The caller already checked that \p I calls the appropriate LibFunc with a
8929/// correct prototype.
8930bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
8931 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
8932 const Value *Size = I.getArgOperand(2);
8933 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
8934 if (CSize && CSize->getZExtValue() == 0) {
8936 I.getType(), true);
8937 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
8938 return true;
8939 }
8940
8942 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
8943 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
8945 if (Res.first.getNode()) {
8946 processIntegerCallValue(I, Res.first, true);
8947 PendingLoads.push_back(Res.second);
8948 return true;
8949 }
8950
8951 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
8952 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
8953 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
8954 return false;
8955
8956 // If the target has a fast compare for the given size, it will return a
8957 // preferred load type for that size. Require that the load VT is legal and
8958 // that the target supports unaligned loads of that type. Otherwise, return
8959 // INVALID.
8960 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
8962 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
8963 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
8964 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
8965 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
8966 // TODO: Check alignment of src and dest ptrs.
8967 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
8968 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
8969 if (!TLI.isTypeLegal(LVT) ||
8970 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
8971 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
8973 }
8974
8975 return LVT;
8976 };
8977
8978 // This turns into unaligned loads. We only do this if the target natively
8979 // supports the MVT we'll be loading or if it is small enough (<= 4) that
8980 // we'll only produce a small number of byte loads.
8981 MVT LoadVT;
8982 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
8983 switch (NumBitsToCompare) {
8984 default:
8985 return false;
8986 case 16:
8987 LoadVT = MVT::i16;
8988 break;
8989 case 32:
8990 LoadVT = MVT::i32;
8991 break;
8992 case 64:
8993 case 128:
8994 case 256:
8995 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
8996 break;
8997 }
8998
8999 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9000 return false;
9001
9002 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
9003 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
9004
9005 // Bitcast to a wide integer type if the loads are vectors.
9006 if (LoadVT.isVector()) {
9007 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
9008 LoadL = DAG.getBitcast(CmpVT, LoadL);
9009 LoadR = DAG.getBitcast(CmpVT, LoadR);
9010 }
9011
9012 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
9013 processIntegerCallValue(I, Cmp, false);
9014 return true;
9015}
9016
9017/// See if we can lower a memchr call into an optimized form. If so, return
9018/// true and lower it. Otherwise return false, and it will be lowered like a
9019/// normal call.
9020/// The caller already checked that \p I calls the appropriate LibFunc with a
9021/// correct prototype.
9022bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9023 const Value *Src = I.getArgOperand(0);
9024 const Value *Char = I.getArgOperand(1);
9025 const Value *Length = I.getArgOperand(2);
9026
9028 std::pair<SDValue, SDValue> Res =
9030 getValue(Src), getValue(Char), getValue(Length),
9031 MachinePointerInfo(Src));
9032 if (Res.first.getNode()) {
9033 setValue(&I, Res.first);
9034 PendingLoads.push_back(Res.second);
9035 return true;
9036 }
9037
9038 return false;
9039}
9040
9041/// See if we can lower a mempcpy call into an optimized form. If so, return
9042/// true and lower it. Otherwise return false, and it will be lowered like a
9043/// normal call.
9044/// The caller already checked that \p I calls the appropriate LibFunc with a
9045/// correct prototype.
9046bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9047 SDValue Dst = getValue(I.getArgOperand(0));
9048 SDValue Src = getValue(I.getArgOperand(1));
9049 SDValue Size = getValue(I.getArgOperand(2));
9050
9051 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
9052 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
9053 // DAG::getMemcpy needs Alignment to be defined.
9054 Align Alignment = std::min(DstAlign, SrcAlign);
9055
9056 SDLoc sdl = getCurSDLoc();
9057
9058 // In the mempcpy context we need to pass in a false value for isTailCall
9059 // because the return pointer needs to be adjusted by the size of
9060 // the copied memory.
9061 SDValue Root = getMemoryRoot();
9062 SDValue MC = DAG.getMemcpy(
9063 Root, sdl, Dst, Src, Size, Alignment, false, false, /*CI=*/nullptr,
9064 std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
9065 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
9066 assert(MC.getNode() != nullptr &&
9067 "** memcpy should not be lowered as TailCall in mempcpy context **");
9068 DAG.setRoot(MC);
9069
9070 // Check if Size needs to be truncated or extended.
9071 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
9072
9073 // Adjust return pointer to point just past the last dst byte.
9074 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
9075 Dst, Size);
9076 setValue(&I, DstPlusSize);
9077 return true;
9078}
9079
9080/// See if we can lower a strcpy call into an optimized form. If so, return
9081/// true and lower it, otherwise return false and it will be lowered like a
9082/// normal call.
9083/// The caller already checked that \p I calls the appropriate LibFunc with a
9084/// correct prototype.
9085bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9086 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9087
9089 std::pair<SDValue, SDValue> Res =
9091 getValue(Arg0), getValue(Arg1),
9092 MachinePointerInfo(Arg0),
9093 MachinePointerInfo(Arg1), isStpcpy);
9094 if (Res.first.getNode()) {
9095 setValue(&I, Res.first);
9096 DAG.setRoot(Res.second);
9097 return true;
9098 }
9099
9100 return false;
9101}
9102
9103/// See if we can lower a strcmp call into an optimized form. If so, return
9104/// true and lower it, otherwise return false and it will be lowered like a
9105/// normal call.
9106/// The caller already checked that \p I calls the appropriate LibFunc with a
9107/// correct prototype.
9108bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9109 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9110
9112 std::pair<SDValue, SDValue> Res =
9114 getValue(Arg0), getValue(Arg1),
9115 MachinePointerInfo(Arg0),
9116 MachinePointerInfo(Arg1));
9117 if (Res.first.getNode()) {
9118 processIntegerCallValue(I, Res.first, true);
9119 PendingLoads.push_back(Res.second);
9120 return true;
9121 }
9122
9123 return false;
9124}
9125
9126/// See if we can lower a strlen call into an optimized form. If so, return
9127/// true and lower it, otherwise return false and it will be lowered like a
9128/// normal call.
9129/// The caller already checked that \p I calls the appropriate LibFunc with a
9130/// correct prototype.
9131bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9132 const Value *Arg0 = I.getArgOperand(0);
9133
9135 std::pair<SDValue, SDValue> Res =
9137 getValue(Arg0), MachinePointerInfo(Arg0));
9138 if (Res.first.getNode()) {
9139 processIntegerCallValue(I, Res.first, false);
9140 PendingLoads.push_back(Res.second);
9141 return true;
9142 }
9143
9144 return false;
9145}
9146
9147/// See if we can lower a strnlen call into an optimized form. If so, return
9148/// true and lower it, otherwise return false and it will be lowered like a
9149/// normal call.
9150/// The caller already checked that \p I calls the appropriate LibFunc with a
9151/// correct prototype.
9152bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9153 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9154
9156 std::pair<SDValue, SDValue> Res =
9158 getValue(Arg0), getValue(Arg1),
9159 MachinePointerInfo(Arg0));
9160 if (Res.first.getNode()) {
9161 processIntegerCallValue(I, Res.first, false);
9162 PendingLoads.push_back(Res.second);
9163 return true;
9164 }
9165
9166 return false;
9167}
9168
9169/// See if we can lower a unary floating-point operation into an SDNode with
9170/// the specified Opcode. If so, return true and lower it, otherwise return
9171/// false and it will be lowered like a normal call.
9172/// The caller already checked that \p I calls the appropriate LibFunc with a
9173/// correct prototype.
9174bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9175 unsigned Opcode) {
9176 // We already checked this call's prototype; verify it doesn't modify errno.
9177 if (!I.onlyReadsMemory())
9178 return false;
9179
9181 Flags.copyFMF(cast<FPMathOperator>(I));
9182
9183 SDValue Tmp = getValue(I.getArgOperand(0));
9184 setValue(&I,
9185 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
9186 return true;
9187}
9188
9189/// See if we can lower a binary floating-point operation into an SDNode with
9190/// the specified Opcode. If so, return true and lower it. Otherwise return
9191/// false, and it will be lowered like a normal call.
9192/// The caller already checked that \p I calls the appropriate LibFunc with a
9193/// correct prototype.
9194bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9195 unsigned Opcode) {
9196 // We already checked this call's prototype; verify it doesn't modify errno.
9197 if (!I.onlyReadsMemory())
9198 return false;
9199
9201 Flags.copyFMF(cast<FPMathOperator>(I));
9202
9203 SDValue Tmp0 = getValue(I.getArgOperand(0));
9204 SDValue Tmp1 = getValue(I.getArgOperand(1));
9205 EVT VT = Tmp0.getValueType();
9206 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
9207 return true;
9208}
9209
9210void SelectionDAGBuilder::visitCall(const CallInst &I) {
9211 // Handle inline assembly differently.
9212 if (I.isInlineAsm()) {
9213 visitInlineAsm(I);
9214 return;
9215 }
9216
9218
9219 if (Function *F = I.getCalledFunction()) {
9220 if (F->isDeclaration()) {
9221 // Is this an LLVM intrinsic or a target-specific intrinsic?
9222 unsigned IID = F->getIntrinsicID();
9223 if (!IID)
9224 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
9225 IID = II->getIntrinsicID(F);
9226
9227 if (IID) {
9228 visitIntrinsicCall(I, IID);
9229 return;
9230 }
9231 }
9232
9233 // Check for well-known libc/libm calls. If the function is internal, it
9234 // can't be a library call. Don't do the check if marked as nobuiltin for
9235 // some reason or the call site requires strict floating point semantics.
9236 LibFunc Func;
9237 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
9238 F->hasName() && LibInfo->getLibFunc(*F, Func) &&
9240 switch (Func) {
9241 default: break;
9242 case LibFunc_bcmp:
9243 if (visitMemCmpBCmpCall(I))
9244 return;
9245 break;
9246 case LibFunc_copysign:
9247 case LibFunc_copysignf:
9248 case LibFunc_copysignl:
9249 // We already checked this call's prototype; verify it doesn't modify
9250 // errno.
9251 if (I.onlyReadsMemory()) {
9252 SDValue LHS = getValue(I.getArgOperand(0));
9253 SDValue RHS = getValue(I.getArgOperand(1));
9255 LHS.getValueType(), LHS, RHS));
9256 return;
9257 }
9258 break;
9259 case LibFunc_fabs:
9260 case LibFunc_fabsf:
9261 case LibFunc_fabsl:
9262 if (visitUnaryFloatCall(I, ISD::FABS))
9263 return;
9264 break;
9265 case LibFunc_fmin:
9266 case LibFunc_fminf:
9267 case LibFunc_fminl:
9268 if (visitBinaryFloatCall(I, ISD::FMINNUM))
9269 return;
9270 break;
9271 case LibFunc_fmax:
9272 case LibFunc_fmaxf:
9273 case LibFunc_fmaxl:
9274 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
9275 return;
9276 break;
9277 case LibFunc_fminimum_num:
9278 case LibFunc_fminimum_numf:
9279 case LibFunc_fminimum_numl:
9280 if (visitBinaryFloatCall(I, ISD::FMINIMUMNUM))
9281 return;
9282 break;
9283 case LibFunc_fmaximum_num:
9284 case LibFunc_fmaximum_numf:
9285 case LibFunc_fmaximum_numl:
9286 if (visitBinaryFloatCall(I, ISD::FMAXIMUMNUM))
9287 return;
9288 break;
9289 case LibFunc_sin:
9290 case LibFunc_sinf:
9291 case LibFunc_sinl:
9292 if (visitUnaryFloatCall(I, ISD::FSIN))
9293 return;
9294 break;
9295 case LibFunc_cos:
9296 case LibFunc_cosf:
9297 case LibFunc_cosl:
9298 if (visitUnaryFloatCall(I, ISD::FCOS))
9299 return;
9300 break;
9301 case LibFunc_tan:
9302 case LibFunc_tanf:
9303 case LibFunc_tanl:
9304 if (visitUnaryFloatCall(I, ISD::FTAN))
9305 return;
9306 break;
9307 case LibFunc_asin:
9308 case LibFunc_asinf:
9309 case LibFunc_asinl:
9310 if (visitUnaryFloatCall(I, ISD::FASIN))
9311 return;
9312 break;
9313 case LibFunc_acos:
9314 case LibFunc_acosf:
9315 case LibFunc_acosl:
9316 if (visitUnaryFloatCall(I, ISD::FACOS))
9317 return;
9318 break;
9319 case LibFunc_atan:
9320 case LibFunc_atanf:
9321 case LibFunc_atanl:
9322 if (visitUnaryFloatCall(I, ISD::FATAN))
9323 return;
9324 break;
9325 case LibFunc_sinh:
9326 case LibFunc_sinhf:
9327 case LibFunc_sinhl:
9328 if (visitUnaryFloatCall(I, ISD::FSINH))
9329 return;
9330 break;
9331 case LibFunc_cosh:
9332 case LibFunc_coshf:
9333 case LibFunc_coshl:
9334 if (visitUnaryFloatCall(I, ISD::FCOSH))
9335 return;
9336 break;
9337 case LibFunc_tanh:
9338 case LibFunc_tanhf:
9339 case LibFunc_tanhl:
9340 if (visitUnaryFloatCall(I, ISD::FTANH))
9341 return;
9342 break;
9343 case LibFunc_sqrt:
9344 case LibFunc_sqrtf:
9345 case LibFunc_sqrtl:
9346 case LibFunc_sqrt_finite:
9347 case LibFunc_sqrtf_finite:
9348 case LibFunc_sqrtl_finite:
9349 if (visitUnaryFloatCall(I, ISD::FSQRT))
9350 return;
9351 break;
9352 case LibFunc_floor:
9353 case LibFunc_floorf:
9354 case LibFunc_floorl:
9355 if (visitUnaryFloatCall(I, ISD::FFLOOR))
9356 return;
9357 break;
9358 case LibFunc_nearbyint:
9359 case LibFunc_nearbyintf:
9360 case LibFunc_nearbyintl:
9361 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
9362 return;
9363 break;
9364 case LibFunc_ceil:
9365 case LibFunc_ceilf:
9366 case LibFunc_ceill:
9367 if (visitUnaryFloatCall(I, ISD::FCEIL))
9368 return;
9369 break;
9370 case LibFunc_rint:
9371 case LibFunc_rintf:
9372 case LibFunc_rintl:
9373 if (visitUnaryFloatCall(I, ISD::FRINT))
9374 return;
9375 break;
9376 case LibFunc_round:
9377 case LibFunc_roundf:
9378 case LibFunc_roundl:
9379 if (visitUnaryFloatCall(I, ISD::FROUND))
9380 return;
9381 break;
9382 case LibFunc_trunc:
9383 case LibFunc_truncf:
9384 case LibFunc_truncl:
9385 if (visitUnaryFloatCall(I, ISD::FTRUNC))
9386 return;
9387 break;
9388 case LibFunc_log2:
9389 case LibFunc_log2f:
9390 case LibFunc_log2l:
9391 if (visitUnaryFloatCall(I, ISD::FLOG2))
9392 return;
9393 break;
9394 case LibFunc_exp2:
9395 case LibFunc_exp2f:
9396 case LibFunc_exp2l:
9397 if (visitUnaryFloatCall(I, ISD::FEXP2))
9398 return;
9399 break;
9400 case LibFunc_exp10:
9401 case LibFunc_exp10f:
9402 case LibFunc_exp10l:
9403 if (visitUnaryFloatCall(I, ISD::FEXP10))
9404 return;
9405 break;
9406 case LibFunc_ldexp:
9407 case LibFunc_ldexpf:
9408 case LibFunc_ldexpl:
9409 if (visitBinaryFloatCall(I, ISD::FLDEXP))
9410 return;
9411 break;
9412 case LibFunc_memcmp:
9413 if (visitMemCmpBCmpCall(I))
9414 return;
9415 break;
9416 case LibFunc_mempcpy:
9417 if (visitMemPCpyCall(I))
9418 return;
9419 break;
9420 case LibFunc_memchr:
9421 if (visitMemChrCall(I))
9422 return;
9423 break;
9424 case LibFunc_strcpy:
9425 if (visitStrCpyCall(I, false))
9426 return;
9427 break;
9428 case LibFunc_stpcpy:
9429 if (visitStrCpyCall(I, true))
9430 return;
9431 break;
9432 case LibFunc_strcmp:
9433 if (visitStrCmpCall(I))
9434 return;
9435 break;
9436 case LibFunc_strlen:
9437 if (visitStrLenCall(I))
9438 return;
9439 break;
9440 case LibFunc_strnlen:
9441 if (visitStrNLenCall(I))
9442 return;
9443 break;
9444 }
9445 }
9446 }
9447
9448 if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
9449 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
9450 return;
9451 }
9452
9453 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9454 // have to do anything here to lower funclet bundles.
9455 // CFGuardTarget bundles are lowered in LowerCallTo.
9456 assert(!I.hasOperandBundlesOtherThan(
9457 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9458 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9459 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9460 LLVMContext::OB_convergencectrl}) &&
9461 "Cannot lower calls with arbitrary operand bundles!");
9462
9463 SDValue Callee = getValue(I.getCalledOperand());
9464
9465 if (I.hasDeoptState())
9466 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
9467 else
9468 // Check if we can potentially perform a tail call. More detailed checking
9469 // is be done within LowerCallTo, after more information about the call is
9470 // known.
9471 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
9472}
9473
9475 const CallBase &CB, const BasicBlock *EHPadBB) {
9476 auto PAB = CB.getOperandBundle("ptrauth");
9477 const Value *CalleeV = CB.getCalledOperand();
9478
9479 // Gather the call ptrauth data from the operand bundle:
9480 // [ i32 <key>, i64 <discriminator> ]
9481 const auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
9482 const Value *Discriminator = PAB->Inputs[1];
9483
9484 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
9485 assert(Discriminator->getType()->isIntegerTy(64) &&
9486 "Invalid ptrauth discriminator");
9487
9488 // Look through ptrauth constants to find the raw callee.
9489 // Do a direct unauthenticated call if we found it and everything matches.
9490 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CalleeV))
9491 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
9492 DAG.getDataLayout()))
9493 return LowerCallTo(CB, getValue(CalleeCPA->getPointer()), CB.isTailCall(),
9494 CB.isMustTailCall(), EHPadBB);
9495
9496 // Functions should never be ptrauth-called directly.
9497 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
9498
9499 // Otherwise, do an authenticated indirect call.
9500 TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
9501 getValue(Discriminator)};
9502
9503 LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
9504 EHPadBB, &PAI);
9505}
9506
9507namespace {
9508
9509/// AsmOperandInfo - This contains information for each constraint that we are
9510/// lowering.
9511class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9512public:
9513 /// CallOperand - If this is the result output operand or a clobber
9514 /// this is null, otherwise it is the incoming operand to the CallInst.
9515 /// This gets modified as the asm is processed.
9516 SDValue CallOperand;
9517
9518 /// AssignedRegs - If this is a register or register class operand, this
9519 /// contains the set of register corresponding to the operand.
9520 RegsForValue AssignedRegs;
9521
9522 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9523 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9524 }
9525
9526 /// Whether or not this operand accesses memory
9527 bool hasMemory(const TargetLowering &TLI) const {
9528 // Indirect operand accesses access memory.
9529 if (isIndirect)
9530 return true;
9531
9532 for (const auto &Code : Codes)
9534 return true;
9535
9536 return false;
9537 }
9538};
9539
9540
9541} // end anonymous namespace
9542
9543/// Make sure that the output operand \p OpInfo and its corresponding input
9544/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9545/// out).
9546static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9547 SDISelAsmOperandInfo &MatchingOpInfo,
9548 SelectionDAG &DAG) {
9549 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9550 return;
9551
9553 const auto &TLI = DAG.getTargetLoweringInfo();
9554
9555 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9556 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
9557 OpInfo.ConstraintVT);
9558 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9559 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
9560 MatchingOpInfo.ConstraintVT);
9561 if ((OpInfo.ConstraintVT.isInteger() !=
9562 MatchingOpInfo.ConstraintVT.isInteger()) ||
9563 (MatchRC.second != InputRC.second)) {
9564 // FIXME: error out in a more elegant fashion
9565 report_fatal_error("Unsupported asm: input constraint"
9566 " with a matching output constraint of"
9567 " incompatible type!");
9568 }
9569 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9570}
9571
9572/// Get a direct memory input to behave well as an indirect operand.
9573/// This may introduce stores, hence the need for a \p Chain.
9574/// \return The (possibly updated) chain.
9575static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9576 SDISelAsmOperandInfo &OpInfo,
9577 SelectionDAG &DAG) {
9578 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9579
9580 // If we don't have an indirect input, put it in the constpool if we can,
9581 // otherwise spill it to a stack slot.
9582 // TODO: This isn't quite right. We need to handle these according to
9583 // the addressing mode that the constraint wants. Also, this may take
9584 // an additional register for the computation and we don't want that
9585 // either.
9586
9587 // If the operand is a float, integer, or vector constant, spill to a
9588 // constant pool entry to get its address.
9589 const Value *OpVal = OpInfo.CallOperandVal;
9590 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
9591 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
9592 OpInfo.CallOperand = DAG.getConstantPool(
9593 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
9594 return Chain;
9595 }
9596
9597 // Otherwise, create a stack slot and emit a store to it before the asm.
9598 Type *Ty = OpVal->getType();
9599 auto &DL = DAG.getDataLayout();
9600 TypeSize TySize = DL.getTypeAllocSize(Ty);
9603 int StackID = 0;
9604 if (TySize.isScalable())
9605 StackID = TFI->getStackIDForScalableVectors();
9606 int SSFI = MF.getFrameInfo().CreateStackObject(TySize.getKnownMinValue(),
9607 DL.getPrefTypeAlign(Ty), false,
9608 nullptr, StackID);
9609 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
9610 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
9612 TLI.getMemValueType(DL, Ty));
9613 OpInfo.CallOperand = StackSlot;
9614
9615 return Chain;
9616}
9617
9618/// GetRegistersForValue - Assign registers (virtual or physical) for the
9619/// specified operand. We prefer to assign virtual registers, to allow the
9620/// register allocator to handle the assignment process. However, if the asm
9621/// uses features that we can't model on machineinstrs, we have SDISel do the
9622/// allocation. This produces generally horrible, but correct, code.
9623///
9624/// OpInfo describes the operand
9625/// RefOpInfo describes the matching operand if any, the operand otherwise
9626static std::optional<unsigned>
9628 SDISelAsmOperandInfo &OpInfo,
9629 SDISelAsmOperandInfo &RefOpInfo) {
9630 LLVMContext &Context = *DAG.getContext();
9631 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9632
9636
9637 // No work to do for memory/address operands.
9638 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9639 OpInfo.ConstraintType == TargetLowering::C_Address)
9640 return std::nullopt;
9641
9642 // If this is a constraint for a single physreg, or a constraint for a
9643 // register class, find it.
9644 unsigned AssignedReg;
9645 const TargetRegisterClass *RC;
9646 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
9647 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
9648 // RC is unset only on failure. Return immediately.
9649 if (!RC)
9650 return std::nullopt;
9651
9652 // Get the actual register value type. This is important, because the user
9653 // may have asked for (e.g.) the AX register in i32 type. We need to
9654 // remember that AX is actually i16 to get the right extension.
9655 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
9656
9657 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9658 // If this is an FP operand in an integer register (or visa versa), or more
9659 // generally if the operand value disagrees with the register class we plan
9660 // to stick it in, fix the operand type.
9661 //
9662 // If this is an input value, the bitcast to the new type is done now.
9663 // Bitcast for output value is done at the end of visitInlineAsm().
9664 if ((OpInfo.Type == InlineAsm::isOutput ||
9665 OpInfo.Type == InlineAsm::isInput) &&
9666 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
9667 // Try to convert to the first EVT that the reg class contains. If the
9668 // types are identical size, use a bitcast to convert (e.g. two differing
9669 // vector types). Note: output bitcast is done at the end of
9670 // visitInlineAsm().
9671 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9672 // Exclude indirect inputs while they are unsupported because the code
9673 // to perform the load is missing and thus OpInfo.CallOperand still
9674 // refers to the input address rather than the pointed-to value.
9675 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
9676 OpInfo.CallOperand =
9677 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
9678 OpInfo.ConstraintVT = RegVT;
9679 // If the operand is an FP value and we want it in integer registers,
9680 // use the corresponding integer type. This turns an f64 value into
9681 // i64, which can be passed with two i32 values on a 32-bit machine.
9682 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9683 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
9684 if (OpInfo.Type == InlineAsm::isInput)
9685 OpInfo.CallOperand =
9686 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
9687 OpInfo.ConstraintVT = VT;
9688 }
9689 }
9690 }
9691
9692 // No need to allocate a matching input constraint since the constraint it's
9693 // matching to has already been allocated.
9694 if (OpInfo.isMatchingInputConstraint())
9695 return std::nullopt;
9696
9697 EVT ValueVT = OpInfo.ConstraintVT;
9698 if (OpInfo.ConstraintVT == MVT::Other)
9699 ValueVT = RegVT;
9700
9701 // Initialize NumRegs.
9702 unsigned NumRegs = 1;
9703 if (OpInfo.ConstraintVT != MVT::Other)
9704 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
9705
9706 // If this is a constraint for a specific physical register, like {r17},
9707 // assign it now.
9708
9709 // If this associated to a specific register, initialize iterator to correct
9710 // place. If virtual, make sure we have enough registers
9711
9712 // Initialize iterator if necessary
9715
9716 // Do not check for single registers.
9717 if (AssignedReg) {
9718 I = std::find(I, RC->end(), AssignedReg);
9719 if (I == RC->end()) {
9720 // RC does not contain the selected register, which indicates a
9721 // mismatch between the register and the required type/bitwidth.
9722 return {AssignedReg};
9723 }
9724 }
9725
9726 for (; NumRegs; --NumRegs, ++I) {
9727 assert(I != RC->end() && "Ran out of registers to allocate!");
9728 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
9729 Regs.push_back(R);
9730 }
9731
9732 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
9733 return std::nullopt;
9734}
9735
9736static unsigned
9738 const std::vector<SDValue> &AsmNodeOperands) {
9739 // Scan until we find the definition we already emitted of this operand.
9740 unsigned CurOp = InlineAsm::Op_FirstOperand;
9741 for (; OperandNo; --OperandNo) {
9742 // Advance to the next operand.
9743 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9744 const InlineAsm::Flag F(OpFlag);
9745 assert(
9746 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
9747 "Skipped past definitions?");
9748 CurOp += F.getNumOperandRegisters() + 1;
9749 }
9750 return CurOp;
9751}
9752
9753namespace {
9754
9755class ExtraFlags {
9756 unsigned Flags = 0;
9757
9758public:
9759 explicit ExtraFlags(const CallBase &Call) {
9760 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9761 if (IA->hasSideEffects())
9763 if (IA->isAlignStack())
9765 if (Call.isConvergent())
9767 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
9768 }
9769
9770 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
9771 // Ideally, we would only check against memory constraints. However, the
9772 // meaning of an Other constraint can be target-specific and we can't easily
9773 // reason about it. Therefore, be conservative and set MayLoad/MayStore
9774 // for Other constraints as well.
9777 if (OpInfo.Type == InlineAsm::isInput)
9779 else if (OpInfo.Type == InlineAsm::isOutput)
9781 else if (OpInfo.Type == InlineAsm::isClobber)
9783 }
9784 }
9785
9786 unsigned get() const { return Flags; }
9787};
9788
9789} // end anonymous namespace
9790
9791static bool isFunction(SDValue Op) {
9792 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
9793 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
9794 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
9795
9796 // In normal "call dllimport func" instruction (non-inlineasm) it force
9797 // indirect access by specifing call opcode. And usually specially print
9798 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
9799 // not do in this way now. (In fact, this is similar with "Data Access"
9800 // action). So here we ignore dllimport function.
9801 if (Fn && !Fn->hasDLLImportStorageClass())
9802 return true;
9803 }
9804 }
9805 return false;
9806}
9807
9808/// visitInlineAsm - Handle a call to an InlineAsm object.
9809void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
9810 const BasicBlock *EHPadBB) {
9811 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9812
9813 /// ConstraintOperands - Information about all of the constraints.
9814 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
9815
9819
9820 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
9821 // AsmDialect, MayLoad, MayStore).
9822 bool HasSideEffect = IA->hasSideEffects();
9823 ExtraFlags ExtraInfo(Call);
9824
9825 for (auto &T : TargetConstraints) {
9826 ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
9827 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
9828
9829 if (OpInfo.CallOperandVal)
9830 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
9831
9832 if (!HasSideEffect)
9833 HasSideEffect = OpInfo.hasMemory(TLI);
9834
9835 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
9836 // FIXME: Could we compute this on OpInfo rather than T?
9837
9838 // Compute the constraint code and ConstraintType to use.
9840
9841 if (T.ConstraintType == TargetLowering::C_Immediate &&
9842 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
9843 // We've delayed emitting a diagnostic like the "n" constraint because
9844 // inlining could cause an integer showing up.
9845 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
9846 "' expects an integer constant "
9847 "expression");
9848
9849 ExtraInfo.update(T);
9850 }
9851
9852 // We won't need to flush pending loads if this asm doesn't touch
9853 // memory and is nonvolatile.
9854 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
9855
9856 bool EmitEHLabels = isa<InvokeInst>(Call);
9857 if (EmitEHLabels) {
9858 assert(EHPadBB && "InvokeInst must have an EHPadBB");
9859 }
9860 bool IsCallBr = isa<CallBrInst>(Call);
9861
9862 if (IsCallBr || EmitEHLabels) {
9863 // If this is a callbr or invoke we need to flush pending exports since
9864 // inlineasm_br and invoke are terminators.
9865 // We need to do this before nodes are glued to the inlineasm_br node.
9866 Chain = getControlRoot();
9867 }
9868
9869 MCSymbol *BeginLabel = nullptr;
9870 if (EmitEHLabels) {
9871 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
9872 }
9873
9874 int OpNo = -1;
9875 SmallVector<StringRef> AsmStrs;
9876 IA->collectAsmStrs(AsmStrs);
9877
9878 // Second pass over the constraints: compute which constraint option to use.
9879 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9880 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
9881 OpNo++;
9882
9883 // If this is an output operand with a matching input operand, look up the
9884 // matching input. If their types mismatch, e.g. one is an integer, the
9885 // other is floating point, or their sizes are different, flag it as an
9886 // error.
9887 if (OpInfo.hasMatchingInput()) {
9888 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
9889 patchMatchingInput(OpInfo, Input, DAG);
9890 }
9891
9892 // Compute the constraint code and ConstraintType to use.
9893 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
9894
9895 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
9896 OpInfo.Type == InlineAsm::isClobber) ||
9897 OpInfo.ConstraintType == TargetLowering::C_Address)
9898 continue;
9899
9900 // In Linux PIC model, there are 4 cases about value/label addressing:
9901 //
9902 // 1: Function call or Label jmp inside the module.
9903 // 2: Data access (such as global variable, static variable) inside module.
9904 // 3: Function call or Label jmp outside the module.
9905 // 4: Data access (such as global variable) outside the module.
9906 //
9907 // Due to current llvm inline asm architecture designed to not "recognize"
9908 // the asm code, there are quite troubles for us to treat mem addressing
9909 // differently for same value/adress used in different instuctions.
9910 // For example, in pic model, call a func may in plt way or direclty
9911 // pc-related, but lea/mov a function adress may use got.
9912 //
9913 // Here we try to "recognize" function call for the case 1 and case 3 in
9914 // inline asm. And try to adjust the constraint for them.
9915 //
9916 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
9917 // label, so here we don't handle jmp function label now, but we need to
9918 // enhance it (especilly in PIC model) if we meet meaningful requirements.
9919 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
9920 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
9922 OpInfo.isIndirect = false;
9923 OpInfo.ConstraintType = TargetLowering::C_Address;
9924 }
9925
9926 // If this is a memory input, and if the operand is not indirect, do what we
9927 // need to provide an address for the memory input.
9928 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
9929 !OpInfo.isIndirect) {
9930 assert((OpInfo.isMultipleAlternative ||
9931 (OpInfo.Type == InlineAsm::isInput)) &&
9932 "Can only indirectify direct input operands!");
9933
9934 // Memory operands really want the address of the value.
9935 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
9936
9937 // There is no longer a Value* corresponding to this operand.
9938 OpInfo.CallOperandVal = nullptr;
9939
9940 // It is now an indirect operand.
9941 OpInfo.isIndirect = true;
9942 }
9943
9944 }
9945
9946 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
9947 std::vector<SDValue> AsmNodeOperands;
9948 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
9949 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
9950 IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
9951
9952 // If we have a !srcloc metadata node associated with it, we want to attach
9953 // this to the ultimately generated inline asm machineinstr. To do this, we
9954 // pass in the third operand as this (potentially null) inline asm MDNode.
9955 const MDNode *SrcLoc = Call.getMetadata("srcloc");
9956 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
9957
9958 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
9959 // bits as operand 3.
9960 AsmNodeOperands.push_back(DAG.getTargetConstant(
9961 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9962
9963 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
9964 // this, assign virtual and physical registers for inputs and otput.
9965 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9966 // Assign Registers.
9967 SDISelAsmOperandInfo &RefOpInfo =
9968 OpInfo.isMatchingInputConstraint()
9969 ? ConstraintOperands[OpInfo.getMatchedOperand()]
9970 : OpInfo;
9971 const auto RegError =
9972 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
9973 if (RegError) {
9976 const char *RegName = TRI.getName(*RegError);
9977 emitInlineAsmError(Call, "register '" + Twine(RegName) +
9978 "' allocated for constraint '" +
9979 Twine(OpInfo.ConstraintCode) +
9980 "' does not match required type");
9981 return;
9982 }
9983
9984 auto DetectWriteToReservedRegister = [&]() {
9987 for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
9989 TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
9990 const char *RegName = TRI.getName(Reg);
9991 emitInlineAsmError(Call, "write to reserved register '" +
9992 Twine(RegName) + "'");
9993 return true;
9994 }
9995 }
9996 return false;
9997 };
9998 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
9999 (OpInfo.Type == InlineAsm::isInput &&
10000 !OpInfo.isMatchingInputConstraint())) &&
10001 "Only address as input operand is allowed.");
10002
10003 switch (OpInfo.Type) {
10005 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10006 const InlineAsm::ConstraintCode ConstraintID =
10007 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10009 "Failed to convert memory constraint code to constraint id.");
10010
10011 // Add information to the INLINEASM node to know about this output.
10013 OpFlags.setMemConstraint(ConstraintID);
10014 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
10015 MVT::i32));
10016 AsmNodeOperands.push_back(OpInfo.CallOperand);
10017 } else {
10018 // Otherwise, this outputs to a register (directly for C_Register /
10019 // C_RegisterClass, and a target-defined fashion for
10020 // C_Immediate/C_Other). Find a register that we can use.
10021 if (OpInfo.AssignedRegs.Regs.empty()) {
10022 emitInlineAsmError(
10023 Call, "couldn't allocate output register for constraint '" +
10024 Twine(OpInfo.ConstraintCode) + "'");
10025 return;
10026 }
10027
10028 if (DetectWriteToReservedRegister())
10029 return;
10030
10031 // Add information to the INLINEASM node to know that this register is
10032 // set.
10033 OpInfo.AssignedRegs.AddInlineAsmOperands(
10034 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10036 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
10037 }
10038 break;
10039
10040 case InlineAsm::isInput:
10041 case InlineAsm::isLabel: {
10042 SDValue InOperandVal = OpInfo.CallOperand;
10043
10044 if (OpInfo.isMatchingInputConstraint()) {
10045 // If this is required to match an output register we have already set,
10046 // just use its register.
10047 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
10048 AsmNodeOperands);
10049 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
10050 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10051 if (OpInfo.isIndirect) {
10052 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10053 emitInlineAsmError(Call, "inline asm not supported yet: "
10054 "don't know how to handle tied "
10055 "indirect register inputs");
10056 return;
10057 }
10058
10063 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
10064 Register TiedReg = R->getReg();
10065 MVT RegVT = R->getSimpleValueType(0);
10066 const TargetRegisterClass *RC =
10067 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)
10068 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
10069 : TRI.getMinimalPhysRegClass(TiedReg);
10070 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
10071 Regs.push_back(MRI.createVirtualRegister(RC));
10072
10073 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10074
10075 SDLoc dl = getCurSDLoc();
10076 // Use the produced MatchedRegs object to
10077 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);
10078 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, true,
10079 OpInfo.getMatchedOperand(), dl, DAG,
10080 AsmNodeOperands);
10081 break;
10082 }
10083
10084 assert(Flag.isMemKind() && "Unknown matching constraint!");
10085 assert(Flag.getNumOperandRegisters() == 1 &&
10086 "Unexpected number of operands");
10087 // Add information to the INLINEASM node to know about this input.
10088 // See InlineAsm.h isUseOperandTiedToDef.
10089 Flag.clearMemConstraint();
10090 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10091 AsmNodeOperands.push_back(DAG.getTargetConstant(
10092 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
10093 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
10094 break;
10095 }
10096
10097 // Treat indirect 'X' constraint as memory.
10098 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10099 OpInfo.isIndirect)
10100 OpInfo.ConstraintType = TargetLowering::C_Memory;
10101
10102 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10103 OpInfo.ConstraintType == TargetLowering::C_Other) {
10104 std::vector<SDValue> Ops;
10105 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
10106 Ops, DAG);
10107 if (Ops.empty()) {
10108 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10109 if (isa<ConstantSDNode>(InOperandVal)) {
10110 emitInlineAsmError(Call, "value out of range for constraint '" +
10111 Twine(OpInfo.ConstraintCode) + "'");
10112 return;
10113 }
10114
10115 emitInlineAsmError(Call,
10116 "invalid operand for inline asm constraint '" +
10117 Twine(OpInfo.ConstraintCode) + "'");
10118 return;
10119 }
10120
10121 // Add information to the INLINEASM node to know about this input.
10122 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10123 AsmNodeOperands.push_back(DAG.getTargetConstant(
10124 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
10125 llvm::append_range(AsmNodeOperands, Ops);
10126 break;
10127 }
10128
10129 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10130 assert((OpInfo.isIndirect ||
10131 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10132 "Operand must be indirect to be a mem!");
10133 assert(InOperandVal.getValueType() ==
10135 "Memory operands expect pointer values");
10136
10137 const InlineAsm::ConstraintCode ConstraintID =
10138 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10140 "Failed to convert memory constraint code to constraint id.");
10141
10142 // Add information to the INLINEASM node to know about this input.
10144 ResOpType.setMemConstraint(ConstraintID);
10145 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
10146 getCurSDLoc(),
10147 MVT::i32));
10148 AsmNodeOperands.push_back(InOperandVal);
10149 break;
10150 }
10151
10152 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10153 const InlineAsm::ConstraintCode ConstraintID =
10154 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10156 "Failed to convert memory constraint code to constraint id.");
10157
10159
10160 SDValue AsmOp = InOperandVal;
10161 if (isFunction(InOperandVal)) {
10162 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
10163 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10164 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
10165 InOperandVal.getValueType(),
10166 GA->getOffset());
10167 }
10168
10169 // Add information to the INLINEASM node to know about this input.
10170 ResOpType.setMemConstraint(ConstraintID);
10171
10172 AsmNodeOperands.push_back(
10173 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
10174
10175 AsmNodeOperands.push_back(AsmOp);
10176 break;
10177 }
10178
10179 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10180 OpInfo.ConstraintType != TargetLowering::C_Register) {
10181 emitInlineAsmError(Call, "unknown asm constraint '" +
10182 Twine(OpInfo.ConstraintCode) + "'");
10183 return;
10184 }
10185
10186 // TODO: Support this.
10187 if (OpInfo.isIndirect) {
10188 emitInlineAsmError(
10189 Call, "Don't know how to handle indirect register inputs yet "
10190 "for constraint '" +
10191 Twine(OpInfo.ConstraintCode) + "'");
10192 return;
10193 }
10194
10195 // Copy the input into the appropriate registers.
10196 if (OpInfo.AssignedRegs.Regs.empty()) {
10197 emitInlineAsmError(Call,
10198 "couldn't allocate input reg for constraint '" +
10199 Twine(OpInfo.ConstraintCode) + "'");
10200 return;
10201 }
10202
10203 if (DetectWriteToReservedRegister())
10204 return;
10205
10206 SDLoc dl = getCurSDLoc();
10207
10208 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,
10209 &Call);
10210
10211 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, false,
10212 0, dl, DAG, AsmNodeOperands);
10213 break;
10214 }
10216 // Add the clobbered value to the operand list, so that the register
10217 // allocator is aware that the physreg got clobbered.
10218 if (!OpInfo.AssignedRegs.Regs.empty())
10219 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::Clobber,
10220 false, 0, getCurSDLoc(), DAG,
10221 AsmNodeOperands);
10222 break;
10223 }
10224 }
10225
10226 // Finish up input operands. Set the input chain and add the flag last.
10227 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10228 if (Glue.getNode()) AsmNodeOperands.push_back(Glue);
10229
10230 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10231 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
10232 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
10233 Glue = Chain.getValue(1);
10234
10235 // Do additional work to generate outputs.
10236
10237 SmallVector<EVT, 1> ResultVTs;
10238 SmallVector<SDValue, 1> ResultValues;
10239 SmallVector<SDValue, 8> OutChains;
10240
10241 llvm::Type *CallResultType = Call.getType();
10242 ArrayRef<Type *> ResultTypes;
10243 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
10244 ResultTypes = StructResult->elements();
10245 else if (!CallResultType->isVoidTy())
10246 ResultTypes = ArrayRef(CallResultType);
10247
10248 auto CurResultType = ResultTypes.begin();
10249 auto handleRegAssign = [&](SDValue V) {
10250 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10251 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10252 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
10253 ++CurResultType;
10254 // If the type of the inline asm call site return value is different but has
10255 // same size as the type of the asm output bitcast it. One example of this
10256 // is for vectors with different width / number of elements. This can
10257 // happen for register classes that can contain multiple different value
10258 // types. The preg or vreg allocated may not have the same VT as was
10259 // expected.
10260 //
10261 // This can also happen for a return value that disagrees with the register
10262 // class it is put in, eg. a double in a general-purpose register on a
10263 // 32-bit machine.
10264 if (ResultVT != V.getValueType() &&
10265 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10266 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
10267 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10268 V.getValueType().isInteger()) {
10269 // If a result value was tied to an input value, the computed result
10270 // may have a wider width than the expected result. Extract the
10271 // relevant portion.
10272 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
10273 }
10274 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10275 ResultVTs.push_back(ResultVT);
10276 ResultValues.push_back(V);
10277 };
10278
10279 // Deal with output operands.
10280 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10281 if (OpInfo.Type == InlineAsm::isOutput) {
10282 SDValue Val;
10283 // Skip trivial output operands.
10284 if (OpInfo.AssignedRegs.Regs.empty())
10285 continue;
10286
10287 switch (OpInfo.ConstraintType) {
10290 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
10291 Chain, &Glue, &Call);
10292 break;
10295 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
10296 OpInfo, DAG);
10297 break;
10299 break; // Already handled.
10301 break; // Silence warning.
10303 assert(false && "Unexpected unknown constraint");
10304 }
10305
10306 // Indirect output manifest as stores. Record output chains.
10307 if (OpInfo.isIndirect) {
10308 const Value *Ptr = OpInfo.CallOperandVal;
10309 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10310 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
10312 OutChains.push_back(Store);
10313 } else {
10314 // generate CopyFromRegs to associated registers.
10315 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10316 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10317 for (const SDValue &V : Val->op_values())
10318 handleRegAssign(V);
10319 } else
10320 handleRegAssign(Val);
10321 }
10322 }
10323 }
10324
10325 // Set results.
10326 if (!ResultValues.empty()) {
10327 assert(CurResultType == ResultTypes.end() &&
10328 "Mismatch in number of ResultTypes");
10329 assert(ResultValues.size() == ResultTypes.size() &&
10330 "Mismatch in number of output operands in asm result");
10331
10333 DAG.getVTList(ResultVTs), ResultValues);
10334 setValue(&Call, V);
10335 }
10336
10337 // Collect store chains.
10338 if (!OutChains.empty())
10339 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
10340
10341 if (EmitEHLabels) {
10342 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
10343 }
10344
10345 // Only Update Root if inline assembly has a memory effect.
10346 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10347 EmitEHLabels)
10348 DAG.setRoot(Chain);
10349}
10350
10351void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10352 const Twine &Message) {
10353 LLVMContext &Ctx = *DAG.getContext();
10354 Ctx.emitError(&Call, Message);
10355
10356 // Make sure we leave the DAG in a valid state
10358 SmallVector<EVT, 1> ValueVTs;
10359 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
10360
10361 if (ValueVTs.empty())
10362 return;
10363
10365 for (const EVT &VT : ValueVTs)
10366 Ops.push_back(DAG.getUNDEF(VT));
10367
10368 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
10369}
10370
10371void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10373 MVT::Other, getRoot(),
10374 getValue(I.getArgOperand(0)),
10375 DAG.getSrcValue(I.getArgOperand(0))));
10376}
10377
10378void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10380 const DataLayout &DL = DAG.getDataLayout();
10382 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
10383 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
10384 DL.getABITypeAlign(I.getType()).value());
10385 DAG.setRoot(V.getValue(1));
10386
10387 if (I.getType()->isPointerTy())
10389 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
10390 setValue(&I, V);
10391}
10392
10393void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10395 MVT::Other, getRoot(),
10396 getValue(I.getArgOperand(0)),
10397 DAG.getSrcValue(I.getArgOperand(0))));
10398}
10399
10400void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10402 MVT::Other, getRoot(),
10403 getValue(I.getArgOperand(0)),
10404 getValue(I.getArgOperand(1)),
10405 DAG.getSrcValue(I.getArgOperand(0)),
10406 DAG.getSrcValue(I.getArgOperand(1))));
10407}
10408
10410 const Instruction &I,
10411 SDValue Op) {
10412 std::optional<ConstantRange> CR = getRange(I);
10413
10414 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
10415 return Op;
10416
10417 APInt Lo = CR->getUnsignedMin();
10418 if (!Lo.isMinValue())
10419 return Op;
10420
10421 APInt Hi = CR->getUnsignedMax();
10422 unsigned Bits = std::max(Hi.getActiveBits(),
10423 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10424
10425 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
10426
10427 SDLoc SL = getCurSDLoc();
10428
10429 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
10430 DAG.getValueType(SmallVT));
10431 unsigned NumVals = Op.getNode()->getNumValues();
10432 if (NumVals == 1)
10433 return ZExt;
10434
10436
10437 Ops.push_back(ZExt);
10438 for (unsigned I = 1; I != NumVals; ++I)
10439 Ops.push_back(Op.getValue(I));
10440
10441 return DAG.getMergeValues(Ops, SL);
10442}
10443
10444/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10445/// the call being lowered.
10446///
10447/// This is a helper for lowering intrinsics that follow a target calling
10448/// convention or require stack pointer adjustment. Only a subset of the
10449/// intrinsic's operands need to participate in the calling convention.
10452 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10453 AttributeSet RetAttrs, bool IsPatchPoint) {
10455 Args.reserve(NumArgs);
10456
10457 // Populate the argument list.
10458 // Attributes for args start at offset 1, after the return attribute.
10459 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10460 ArgI != ArgE; ++ArgI) {
10461 const Value *V = Call->getOperand(ArgI);
10462
10463 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10464
10466 Entry.Node = getValue(V);
10467 Entry.Ty = V->getType();
10468 Entry.setAttributes(Call, ArgI);
10469 Args.push_back(Entry);
10470 }
10471
10473 .setChain(getRoot())
10474 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
10475 RetAttrs)
10476 .setDiscardResult(Call->use_empty())
10477 .setIsPatchPoint(IsPatchPoint)
10479 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
10480}
10481
10482/// Add a stack map intrinsic call's live variable operands to a stackmap
10483/// or patchpoint target node's operand list.
10484///
10485/// Constants are converted to TargetConstants purely as an optimization to
10486/// avoid constant materialization and register allocation.
10487///
10488/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10489/// generate addess computation nodes, and so FinalizeISel can convert the
10490/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10491/// address materialization and register allocation, but may also be required
10492/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10493/// alloca in the entry block, then the runtime may assume that the alloca's
10494/// StackMap location can be read immediately after compilation and that the
10495/// location is valid at any point during execution (this is similar to the
10496/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10497/// only available in a register, then the runtime would need to trap when
10498/// execution reaches the StackMap in order to read the alloca's location.
10499static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10500 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
10501 SelectionDAGBuilder &Builder) {
10502 SelectionDAG &DAG = Builder.DAG;
10503 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10504 SDValue Op = Builder.getValue(Call.getArgOperand(I));
10505
10506 // Things on the stack are pointer-typed, meaning that they are already
10507 // legal and can be emitted directly to target nodes.
10508 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
10509 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
10510 } else {
10511 // Otherwise emit a target independent node to be legalised.
10512 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
10513 }
10514 }
10515}
10516
10517/// Lower llvm.experimental.stackmap.
10518void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10519 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10520 // [live variables...])
10521
10522 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10523
10524 SDValue Chain, InGlue, Callee;
10526
10527 SDLoc DL = getCurSDLoc();
10529
10530 // The stackmap intrinsic only records the live variables (the arguments
10531 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10532 // intrinsic, this won't be lowered to a function call. This means we don't
10533 // have to worry about calling conventions and target specific lowering code.
10534 // Instead we perform the call lowering right here.
10535 //
10536 // chain, flag = CALLSEQ_START(chain, 0, 0)
10537 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10538 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10539 //
10540 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
10541 InGlue = Chain.getValue(1);
10542
10543 // Add the STACKMAP operands, starting with DAG house-keeping.
10544 Ops.push_back(Chain);
10545 Ops.push_back(InGlue);
10546
10547 // Add the <id>, <numShadowBytes> operands.
10548 //
10549 // These do not require legalisation, and can be emitted directly to target
10550 // constant nodes.
10552 assert(ID.getValueType() == MVT::i64);
10553 SDValue IDConst =
10554 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
10555 Ops.push_back(IDConst);
10556
10557 SDValue Shad = getValue(CI.getArgOperand(1));
10558 assert(Shad.getValueType() == MVT::i32);
10559 SDValue ShadConst =
10561 Ops.push_back(ShadConst);
10562
10563 // Add the live variables.
10564 addStackMapLiveVars(CI, 2, DL, Ops, *this);
10565
10566 // Create the STACKMAP node.
10567 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10568 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
10569 InGlue = Chain.getValue(1);
10570
10571 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
10572
10573 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10574
10575 // Set the root to the target-lowered call chain.
10576 DAG.setRoot(Chain);
10577
10578 // Inform the Frame Information that we have a stackmap in this function.
10580}
10581
10582/// Lower llvm.experimental.patchpoint directly to its target opcode.
10583void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10584 const BasicBlock *EHPadBB) {
10585 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
10586 // i32 <numBytes>,
10587 // i8* <target>,
10588 // i32 <numArgs>,
10589 // [Args...],
10590 // [live variables...])
10591
10593 bool IsAnyRegCC = CC == CallingConv::AnyReg;
10594 bool HasDef = !CB.getType()->isVoidTy();
10595 SDLoc dl = getCurSDLoc();
10597
10598 // Handle immediate and symbolic callees.
10599 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
10600 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
10601 /*isTarget=*/true);
10602 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
10603 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
10604 SDLoc(SymbolicCallee),
10605 SymbolicCallee->getValueType(0));
10606
10607 // Get the real number of arguments participating in the call <numArgs>
10609 unsigned NumArgs = NArgVal->getAsZExtVal();
10610
10611 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
10612 // Intrinsics include all meta-operands up to but not including CC.
10613 unsigned NumMetaOpers = PatchPointOpers::CCPos;
10614 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
10615 "Not enough arguments provided to the patchpoint intrinsic");
10616
10617 // For AnyRegCC the arguments are lowered later on manually.
10618 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10619 Type *ReturnTy =
10620 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
10621
10623 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
10624 ReturnTy, CB.getAttributes().getRetAttrs(), true);
10625 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
10626
10627 SDNode *CallEnd = Result.second.getNode();
10628 if (CallEnd->getOpcode() == ISD::EH_LABEL)
10629 CallEnd = CallEnd->getOperand(0).getNode();
10630 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
10631 CallEnd = CallEnd->getOperand(0).getNode();
10632
10633 /// Get a call instruction from the call sequence chain.
10634 /// Tail calls are not allowed.
10635 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
10636 "Expected a callseq node.");
10637 SDNode *Call = CallEnd->getOperand(0).getNode();
10638 bool HasGlue = Call->getGluedNode();
10639
10640 // Replace the target specific call node with the patchable intrinsic.
10642
10643 // Push the chain.
10644 Ops.push_back(*(Call->op_begin()));
10645
10646 // Optionally, push the glue (if any).
10647 if (HasGlue)
10648 Ops.push_back(*(Call->op_end() - 1));
10649
10650 // Push the register mask info.
10651 if (HasGlue)
10652 Ops.push_back(*(Call->op_end() - 2));
10653 else
10654 Ops.push_back(*(Call->op_end() - 1));
10655
10656 // Add the <id> and <numBytes> constants.
10658 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
10660 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
10661
10662 // Add the callee.
10663 Ops.push_back(Callee);
10664
10665 // Adjust <numArgs> to account for any arguments that have been passed on the
10666 // stack instead.
10667 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
10668 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
10669 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10670 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
10671
10672 // Add the calling convention
10673 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
10674
10675 // Add the arguments we omitted previously. The register allocator should
10676 // place these in any free register.
10677 if (IsAnyRegCC)
10678 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
10679 Ops.push_back(getValue(CB.getArgOperand(i)));
10680
10681 // Push the arguments from the call instruction.
10682 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
10683 Ops.append(Call->op_begin() + 2, e);
10684
10685 // Push live variables for the stack map.
10686 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
10687
10688 SDVTList NodeTys;
10689 if (IsAnyRegCC && HasDef) {
10690 // Create the return types based on the intrinsic definition
10692 SmallVector<EVT, 3> ValueVTs;
10693 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
10694 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
10695
10696 // There is always a chain and a glue type at the end
10697 ValueVTs.push_back(MVT::Other);
10698 ValueVTs.push_back(MVT::Glue);
10699 NodeTys = DAG.getVTList(ValueVTs);
10700 } else
10701 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10702
10703 // Replace the target specific call node with a PATCHPOINT node.
10704 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
10705
10706 // Update the NodeMap.
10707 if (HasDef) {
10708 if (IsAnyRegCC)
10709 setValue(&CB, SDValue(PPV.getNode(), 0));
10710 else
10711 setValue(&CB, Result.first);
10712 }
10713
10714 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
10715 // call sequence. Furthermore the location of the chain and glue can change
10716 // when the AnyReg calling convention is used and the intrinsic returns a
10717 // value.
10718 if (IsAnyRegCC && HasDef) {
10719 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
10720 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
10722 } else
10723 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
10724 DAG.DeleteNode(Call);
10725
10726 // Inform the Frame Information that we have a patchpoint in this function.
10728}
10729
10730void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
10731 unsigned Intrinsic) {
10733 SDValue Op1 = getValue(I.getArgOperand(0));
10734 SDValue Op2;
10735 if (I.arg_size() > 1)
10736 Op2 = getValue(I.getArgOperand(1));
10737 SDLoc dl = getCurSDLoc();
10738 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
10739 SDValue Res;
10740 SDNodeFlags SDFlags;
10741 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
10742 SDFlags.copyFMF(*FPMO);
10743
10744 switch (Intrinsic) {
10745 case Intrinsic::vector_reduce_fadd:
10746 if (SDFlags.hasAllowReassociation())
10747 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
10748 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
10749 SDFlags);
10750 else
10751 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
10752 break;
10753 case Intrinsic::vector_reduce_fmul:
10754 if (SDFlags.hasAllowReassociation())
10755 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
10756 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
10757 SDFlags);
10758 else
10759 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
10760 break;
10761 case Intrinsic::vector_reduce_add:
10762 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
10763 break;
10764 case Intrinsic::vector_reduce_mul:
10765 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
10766 break;
10767 case Intrinsic::vector_reduce_and:
10768 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
10769 break;
10770 case Intrinsic::vector_reduce_or:
10771 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
10772 break;
10773 case Intrinsic::vector_reduce_xor:
10774 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
10775 break;
10776 case Intrinsic::vector_reduce_smax:
10777 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
10778 break;
10779 case Intrinsic::vector_reduce_smin:
10780 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
10781 break;
10782 case Intrinsic::vector_reduce_umax:
10783 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
10784 break;
10785 case Intrinsic::vector_reduce_umin:
10786 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
10787 break;
10788 case Intrinsic::vector_reduce_fmax:
10789 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
10790 break;
10791 case Intrinsic::vector_reduce_fmin:
10792 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
10793 break;
10794 case Intrinsic::vector_reduce_fmaximum:
10795 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
10796 break;
10797 case Intrinsic::vector_reduce_fminimum:
10798 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
10799 break;
10800 default:
10801 llvm_unreachable("Unhandled vector reduce intrinsic");
10802 }
10803 setValue(&I, Res);
10804}
10805
10806/// Returns an AttributeList representing the attributes applied to the return
10807/// value of the given call.
10810 if (CLI.RetSExt)
10811 Attrs.push_back(Attribute::SExt);
10812 if (CLI.RetZExt)
10813 Attrs.push_back(Attribute::ZExt);
10814 if (CLI.IsInReg)
10815 Attrs.push_back(Attribute::InReg);
10816
10818 Attrs);
10819}
10820
10821/// TargetLowering::LowerCallTo - This is the default LowerCallTo
10822/// implementation, which just calls LowerCall.
10823/// FIXME: When all targets are
10824/// migrated to using LowerCall, this hook should be integrated into SDISel.
10825std::pair<SDValue, SDValue>
10827 // Handle the incoming return values from the call.
10828 CLI.Ins.clear();
10829 Type *OrigRetTy = CLI.RetTy;
10830 SmallVector<EVT, 4> RetTys;
10832 auto &DL = CLI.DAG.getDataLayout();
10833 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
10834
10835 if (CLI.IsPostTypeLegalization) {
10836 // If we are lowering a libcall after legalization, split the return type.
10837 SmallVector<EVT, 4> OldRetTys;
10838 SmallVector<TypeSize, 4> OldOffsets;
10839 RetTys.swap(OldRetTys);
10840 Offsets.swap(OldOffsets);
10841
10842 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
10843 EVT RetVT = OldRetTys[i];
10844 uint64_t Offset = OldOffsets[i];
10845 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
10846 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
10847 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
10848 RetTys.append(NumRegs, RegisterVT);
10849 for (unsigned j = 0; j != NumRegs; ++j)
10850 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
10851 }
10852 }
10853
10855 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
10856
10857 bool CanLowerReturn =
10859 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
10860
10861 SDValue DemoteStackSlot;
10862 int DemoteStackIdx = -100;
10863 if (!CanLowerReturn) {
10864 // FIXME: equivalent assert?
10865 // assert(!CS.hasInAllocaArgument() &&
10866 // "sret demotion is incompatible with inalloca");
10867 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
10868 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
10870 DemoteStackIdx =
10871 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
10872 Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
10873 DL.getAllocaAddrSpace());
10874
10875 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
10876 ArgListEntry Entry;
10877 Entry.Node = DemoteStackSlot;
10878 Entry.Ty = StackSlotPtrType;
10879 Entry.IsSExt = false;
10880 Entry.IsZExt = false;
10881 Entry.IsInReg = false;
10882 Entry.IsSRet = true;
10883 Entry.IsNest = false;
10884 Entry.IsByVal = false;
10885 Entry.IsByRef = false;
10886 Entry.IsReturned = false;
10887 Entry.IsSwiftSelf = false;
10888 Entry.IsSwiftAsync = false;
10889 Entry.IsSwiftError = false;
10890 Entry.IsCFGuardTarget = false;
10891 Entry.Alignment = Alignment;
10892 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
10893 CLI.NumFixedArgs += 1;
10894 CLI.getArgs()[0].IndirectType = CLI.RetTy;
10895 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
10896
10897 // sret demotion isn't compatible with tail-calls, since the sret argument
10898 // points into the callers stack frame.
10899 CLI.IsTailCall = false;
10900 } else {
10901 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10902 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
10903 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
10904 ISD::ArgFlagsTy Flags;
10905 if (NeedsRegBlock) {
10906 Flags.setInConsecutiveRegs();
10907 if (I == RetTys.size() - 1)
10908 Flags.setInConsecutiveRegsLast();
10909 }
10910 EVT VT = RetTys[I];
10912 CLI.CallConv, VT);
10913 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10914 CLI.CallConv, VT);
10915 for (unsigned i = 0; i != NumRegs; ++i) {
10916 ISD::InputArg MyFlags;
10917 MyFlags.Flags = Flags;
10918 MyFlags.VT = RegisterVT;
10919 MyFlags.ArgVT = VT;
10920 MyFlags.Used = CLI.IsReturnValueUsed;
10921 if (CLI.RetTy->isPointerTy()) {
10922 MyFlags.Flags.setPointer();
10923 MyFlags.Flags.setPointerAddrSpace(
10924 cast<PointerType>(CLI.RetTy)->getAddressSpace());
10925 }
10926 if (CLI.RetSExt)
10927 MyFlags.Flags.setSExt();
10928 if (CLI.RetZExt)
10929 MyFlags.Flags.setZExt();
10930 if (CLI.IsInReg)
10931 MyFlags.Flags.setInReg();
10932 CLI.Ins.push_back(MyFlags);
10933 }
10934 }
10935 }
10936
10937 // We push in swifterror return as the last element of CLI.Ins.
10938 ArgListTy &Args = CLI.getArgs();
10939 if (supportSwiftError()) {
10940 for (const ArgListEntry &Arg : Args) {
10941 if (Arg.IsSwiftError) {
10942 ISD::InputArg MyFlags;
10943 MyFlags.VT = getPointerTy(DL);
10944 MyFlags.ArgVT = EVT(getPointerTy(DL));
10945 MyFlags.Flags.setSwiftError();
10946 CLI.Ins.push_back(MyFlags);
10947 }
10948 }
10949 }
10950
10951 // Handle all of the outgoing arguments.
10952 CLI.Outs.clear();
10953 CLI.OutVals.clear();
10954 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
10955 SmallVector<EVT, 4> ValueVTs;
10956 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
10957 // FIXME: Split arguments if CLI.IsPostTypeLegalization
10958 Type *FinalType = Args[i].Ty;
10959 if (Args[i].IsByVal)
10960 FinalType = Args[i].IndirectType;
10961 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10962 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
10963 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
10964 ++Value) {
10965 EVT VT = ValueVTs[Value];
10966 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
10967 SDValue Op = SDValue(Args[i].Node.getNode(),
10968 Args[i].Node.getResNo() + Value);
10969 ISD::ArgFlagsTy Flags;
10970
10971 // Certain targets (such as MIPS), may have a different ABI alignment
10972 // for a type depending on the context. Give the target a chance to
10973 // specify the alignment it wants.
10974 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
10975 Flags.setOrigAlign(OriginalAlignment);
10976
10977 if (Args[i].Ty->isPointerTy()) {
10978 Flags.setPointer();
10979 Flags.setPointerAddrSpace(
10980 cast<PointerType>(Args[i].Ty)->getAddressSpace());
10981 }
10982 if (Args[i].IsZExt)
10983 Flags.setZExt();
10984 if (Args[i].IsSExt)
10985 Flags.setSExt();
10986 if (Args[i].IsInReg) {
10987 // If we are using vectorcall calling convention, a structure that is
10988 // passed InReg - is surely an HVA
10990 isa<StructType>(FinalType)) {
10991 // The first value of a structure is marked
10992 if (0 == Value)
10993 Flags.setHvaStart();
10994 Flags.setHva();
10995 }
10996 // Set InReg Flag
10997 Flags.setInReg();
10998 }
10999 if (Args[i].IsSRet)
11000 Flags.setSRet();
11001 if (Args[i].IsSwiftSelf)
11002 Flags.setSwiftSelf();
11003 if (Args[i].IsSwiftAsync)
11004 Flags.setSwiftAsync();
11005 if (Args[i].IsSwiftError)
11006 Flags.setSwiftError();
11007 if (Args[i].IsCFGuardTarget)
11008 Flags.setCFGuardTarget();
11009 if (Args[i].IsByVal)
11010 Flags.setByVal();
11011 if (Args[i].IsByRef)
11012 Flags.setByRef();
11013 if (Args[i].IsPreallocated) {
11014 Flags.setPreallocated();
11015 // Set the byval flag for CCAssignFn callbacks that don't know about
11016 // preallocated. This way we can know how many bytes we should've
11017 // allocated and how many bytes a callee cleanup function will pop. If
11018 // we port preallocated to more targets, we'll have to add custom
11019 // preallocated handling in the various CC lowering callbacks.
11020 Flags.setByVal();
11021 }
11022 if (Args[i].IsInAlloca) {
11023 Flags.setInAlloca();
11024 // Set the byval flag for CCAssignFn callbacks that don't know about
11025 // inalloca. This way we can know how many bytes we should've allocated
11026 // and how many bytes a callee cleanup function will pop. If we port
11027 // inalloca to more targets, we'll have to add custom inalloca handling
11028 // in the various CC lowering callbacks.
11029 Flags.setByVal();
11030 }
11031 Align MemAlign;
11032 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11033 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
11034 Flags.setByValSize(FrameSize);
11035
11036 // info is not there but there are cases it cannot get right.
11037 if (auto MA = Args[i].Alignment)
11038 MemAlign = *MA;
11039 else
11040 MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
11041 } else if (auto MA = Args[i].Alignment) {
11042 MemAlign = *MA;
11043 } else {
11044 MemAlign = OriginalAlignment;
11045 }
11046 Flags.setMemAlign(MemAlign);
11047 if (Args[i].IsNest)
11048 Flags.setNest();
11049 if (NeedsRegBlock)
11050 Flags.setInConsecutiveRegs();
11051
11053 CLI.CallConv, VT);
11054 unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
11055 CLI.CallConv, VT);
11056 SmallVector<SDValue, 4> Parts(NumParts);
11057 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
11058
11059 if (Args[i].IsSExt)
11060 ExtendKind = ISD::SIGN_EXTEND;
11061 else if (Args[i].IsZExt)
11062 ExtendKind = ISD::ZERO_EXTEND;
11063
11064 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11065 // for now.
11066 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11068 assert((CLI.RetTy == Args[i].Ty ||
11069 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11071 Args[i].Ty->getPointerAddressSpace())) &&
11072 RetTys.size() == NumValues && "unexpected use of 'returned'");
11073 // Before passing 'returned' to the target lowering code, ensure that
11074 // either the register MVT and the actual EVT are the same size or that
11075 // the return value and argument are extended in the same way; in these
11076 // cases it's safe to pass the argument register value unchanged as the
11077 // return register value (although it's at the target's option whether
11078 // to do so)
11079 // TODO: allow code generation to take advantage of partially preserved
11080 // registers rather than clobbering the entire register when the
11081 // parameter extension method is not compatible with the return
11082 // extension method
11083 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11084 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11085 CLI.RetZExt == Args[i].IsZExt))
11086 Flags.setReturned();
11087 }
11088
11089 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
11090 CLI.CallConv, ExtendKind);
11091
11092 for (unsigned j = 0; j != NumParts; ++j) {
11093 // if it isn't first piece, alignment must be 1
11094 // For scalable vectors the scalable part is currently handled
11095 // by individual targets, so we just use the known minimum size here.
11096 ISD::OutputArg MyFlags(
11097 Flags, Parts[j].getValueType().getSimpleVT(), VT,
11098 i < CLI.NumFixedArgs, i,
11099 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11100 if (NumParts > 1 && j == 0)
11101 MyFlags.Flags.setSplit();
11102 else if (j != 0) {
11103 MyFlags.Flags.setOrigAlign(Align(1));
11104 if (j == NumParts - 1)
11105 MyFlags.Flags.setSplitEnd();
11106 }
11107
11108 CLI.Outs.push_back(MyFlags);
11109 CLI.OutVals.push_back(Parts[j]);
11110 }
11111
11112 if (NeedsRegBlock && Value == NumValues - 1)
11113 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11114 }
11115 }
11116
11118 CLI.Chain = LowerCall(CLI, InVals);
11119
11120 // Update CLI.InVals to use outside of this function.
11121 CLI.InVals = InVals;
11122
11123 // Verify that the target's LowerCall behaved as expected.
11124 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11125 "LowerCall didn't return a valid chain!");
11126 assert((!CLI.IsTailCall || InVals.empty()) &&
11127 "LowerCall emitted a return value for a tail call!");
11128 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11129 "LowerCall didn't emit the correct number of values!");
11130
11131 // For a tail call, the return value is merely live-out and there aren't
11132 // any nodes in the DAG representing it. Return a special value to
11133 // indicate that a tail call has been emitted and no more Instructions
11134 // should be processed in the current block.
11135 if (CLI.IsTailCall) {
11136 CLI.DAG.setRoot(CLI.Chain);
11137 return std::make_pair(SDValue(), SDValue());
11138 }
11139
11140#ifndef NDEBUG
11141 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11142 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11143 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11144 "LowerCall emitted a value with the wrong type!");
11145 }
11146#endif
11147
11148 SmallVector<SDValue, 4> ReturnValues;
11149 if (!CanLowerReturn) {
11150 // The instruction result is the result of loading from the
11151 // hidden sret parameter.
11153 Type *PtrRetTy =
11154 PointerType::get(OrigRetTy->getContext(), DL.getAllocaAddrSpace());
11155
11156 ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
11157 assert(PVTs.size() == 1 && "Pointers should fit in one register");
11158 EVT PtrVT = PVTs[0];
11159
11160 unsigned NumValues = RetTys.size();
11161 ReturnValues.resize(NumValues);
11162 SmallVector<SDValue, 4> Chains(NumValues);
11163
11164 // An aggregate return value cannot wrap around the address space, so
11165 // offsets to its parts don't wrap either.
11166 SDNodeFlags Flags;
11167 Flags.setNoUnsignedWrap(true);
11168
11170 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
11171 for (unsigned i = 0; i < NumValues; ++i) {
11172 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
11173 CLI.DAG.getConstant(Offsets[i], CLI.DL,
11174 PtrVT), Flags);
11175 SDValue L = CLI.DAG.getLoad(
11176 RetTys[i], CLI.DL, CLI.Chain, Add,
11178 DemoteStackIdx, Offsets[i]),
11179 HiddenSRetAlign);
11180 ReturnValues[i] = L;
11181 Chains[i] = L.getValue(1);
11182 }
11183
11184 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
11185 } else {
11186 // Collect the legal value parts into potentially illegal values
11187 // that correspond to the original function's return values.
11188 std::optional<ISD::NodeType> AssertOp;
11189 if (CLI.RetSExt)
11190 AssertOp = ISD::AssertSext;
11191 else if (CLI.RetZExt)
11192 AssertOp = ISD::AssertZext;
11193 unsigned CurReg = 0;
11194 for (EVT VT : RetTys) {
11196 CLI.CallConv, VT);
11197 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
11198 CLI.CallConv, VT);
11199
11200 ReturnValues.push_back(getCopyFromParts(
11201 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
11202 CLI.Chain, CLI.CallConv, AssertOp));
11203 CurReg += NumRegs;
11204 }
11205
11206 // For a function returning void, there is no return value. We can't create
11207 // such a node, so we just return a null return value in that case. In
11208 // that case, nothing will actually look at the value.
11209 if (ReturnValues.empty())
11210 return std::make_pair(SDValue(), CLI.Chain);
11211 }
11212
11213 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
11214 CLI.DAG.getVTList(RetTys), ReturnValues);
11215 return std::make_pair(Res, CLI.Chain);
11216}
11217
11218/// Places new result values for the node in Results (their number
11219/// and types must exactly match those of the original return values of
11220/// the node), or leaves Results empty, which indicates that the node is not
11221/// to be custom lowered after all.
11224 SelectionDAG &DAG) const {
11225 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
11226
11227 if (!Res.getNode())
11228 return;
11229
11230 // If the original node has one result, take the return value from
11231 // LowerOperation as is. It might not be result number 0.
11232 if (N->getNumValues() == 1) {
11233 Results.push_back(Res);
11234 return;
11235 }
11236
11237 // If the original node has multiple results, then the return node should
11238 // have the same number of results.
11239 assert((N->getNumValues() == Res->getNumValues()) &&
11240 "Lowering returned the wrong number of results!");
11241
11242 // Places new result values base on N result number.
11243 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11244 Results.push_back(Res.getValue(I));
11245}
11246
11248 llvm_unreachable("LowerOperation not implemented for this target!");
11249}
11250
11252 unsigned Reg,
11253 ISD::NodeType ExtendType) {
11255 assert((Op.getOpcode() != ISD::CopyFromReg ||
11256 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11257 "Copy from a reg to the same reg!");
11258 assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
11259
11261 // If this is an InlineAsm we have to match the registers required, not the
11262 // notional registers required by the type.
11263
11264 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11265 std::nullopt); // This is not an ABI copy.
11266 SDValue Chain = DAG.getEntryNode();
11267
11268 if (ExtendType == ISD::ANY_EXTEND) {
11269 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
11270 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11271 ExtendType = PreferredExtendIt->second;
11272 }
11273 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
11274 PendingExports.push_back(Chain);
11275}
11276
11278
11279/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11280/// entry block, return true. This includes arguments used by switches, since
11281/// the switch may expand into multiple basic blocks.
11282static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11283 // With FastISel active, we may be splitting blocks, so force creation
11284 // of virtual registers for all non-dead arguments.
11285 if (FastISel)
11286 return A->use_empty();
11287
11288 const BasicBlock &Entry = A->getParent()->front();
11289 for (const User *U : A->users())
11290 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
11291 return false; // Use not in entry block.
11292
11293 return true;
11294}
11295
11297 DenseMap<const Argument *,
11298 std::pair<const AllocaInst *, const StoreInst *>>;
11299
11300/// Scan the entry block of the function in FuncInfo for arguments that look
11301/// like copies into a local alloca. Record any copied arguments in
11302/// ArgCopyElisionCandidates.
11303static void
11305 FunctionLoweringInfo *FuncInfo,
11306 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
11307 // Record the state of every static alloca used in the entry block. Argument
11308 // allocas are all used in the entry block, so we need approximately as many
11309 // entries as we have arguments.
11310 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
11312 unsigned NumArgs = FuncInfo->Fn->arg_size();
11313 StaticAllocas.reserve(NumArgs * 2);
11314
11315 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
11316 if (!V)
11317 return nullptr;
11318 V = V->stripPointerCasts();
11319 const auto *AI = dyn_cast<AllocaInst>(V);
11320 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
11321 return nullptr;
11322 auto Iter = StaticAllocas.insert({AI, Unknown});
11323 return &Iter.first->second;
11324 };
11325
11326 // Look for stores of arguments to static allocas. Look through bitcasts and
11327 // GEPs to handle type coercions, as long as the alloca is fully initialized
11328 // by the store. Any non-store use of an alloca escapes it and any subsequent
11329 // unanalyzed store might write it.
11330 // FIXME: Handle structs initialized with multiple stores.
11331 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11332 // Look for stores, and handle non-store uses conservatively.
11333 const auto *SI = dyn_cast<StoreInst>(&I);
11334 if (!SI) {
11335 // We will look through cast uses, so ignore them completely.
11336 if (I.isCast())
11337 continue;
11338 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11339 // to allocas.
11340 if (I.isDebugOrPseudoInst())
11341 continue;
11342 // This is an unknown instruction. Assume it escapes or writes to all
11343 // static alloca operands.
11344 for (const Use &U : I.operands()) {
11345 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11346 *Info = StaticAllocaInfo::Clobbered;
11347 }
11348 continue;
11349 }
11350
11351 // If the stored value is a static alloca, mark it as escaped.
11352 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11353 *Info = StaticAllocaInfo::Clobbered;
11354
11355 // Check if the destination is a static alloca.
11356 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11357 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11358 if (!Info)
11359 continue;
11360 const AllocaInst *AI = cast<AllocaInst>(Dst);
11361
11362 // Skip allocas that have been initialized or clobbered.
11363 if (*Info != StaticAllocaInfo::Unknown)
11364 continue;
11365
11366 // Check if the stored value is an argument, and that this store fully
11367 // initializes the alloca.
11368 // If the argument type has padding bits we can't directly forward a pointer
11369 // as the upper bits may contain garbage.
11370 // Don't elide copies from the same argument twice.
11371 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11372 const auto *Arg = dyn_cast<Argument>(Val);
11373 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11374 Arg->getType()->isEmptyTy() ||
11375 DL.getTypeStoreSize(Arg->getType()) !=
11376 DL.getTypeAllocSize(AI->getAllocatedType()) ||
11377 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
11378 ArgCopyElisionCandidates.count(Arg)) {
11379 *Info = StaticAllocaInfo::Clobbered;
11380 continue;
11381 }
11382
11383 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11384 << '\n');
11385
11386 // Mark this alloca and store for argument copy elision.
11387 *Info = StaticAllocaInfo::Elidable;
11388 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
11389
11390 // Stop scanning if we've seen all arguments. This will happen early in -O0
11391 // builds, which is useful, because -O0 builds have large entry blocks and
11392 // many allocas.
11393 if (ArgCopyElisionCandidates.size() == NumArgs)
11394 break;
11395 }
11396}
11397
11398/// Try to elide argument copies from memory into a local alloca. Succeeds if
11399/// ArgVal is a load from a suitable fixed stack object.
11402 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11403 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11404 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11405 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11406 // Check if this is a load from a fixed stack object.
11407 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
11408 if (!LNode)
11409 return;
11410 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
11411 if (!FINode)
11412 return;
11413
11414 // Check that the fixed stack object is the right size and alignment.
11415 // Look at the alignment that the user wrote on the alloca instead of looking
11416 // at the stack object.
11417 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
11418 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11419 const AllocaInst *AI = ArgCopyIter->second.first;
11420 int FixedIndex = FINode->getIndex();
11421 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11422 int OldIndex = AllocaIndex;
11423 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11424 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
11425 LLVM_DEBUG(
11426 dbgs() << " argument copy elision failed due to bad fixed stack "
11427 "object size\n");
11428 return;
11429 }
11430 Align RequiredAlignment = AI->getAlign();
11431 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
11432 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11433 "greater than stack argument alignment ("
11434 << DebugStr(RequiredAlignment) << " vs "
11435 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11436 return;
11437 }
11438
11439 // Perform the elision. Delete the old stack object and replace its only use
11440 // in the variable info map. Mark the stack object as mutable and aliased.
11441 LLVM_DEBUG({
11442 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11443 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11444 << '\n';
11445 });
11446 MFI.RemoveStackObject(OldIndex);
11447 MFI.setIsImmutableObjectIndex(FixedIndex, false);
11448 MFI.setIsAliasedObjectIndex(FixedIndex, true);
11449 AllocaIndex = FixedIndex;
11450 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
11451 for (SDValue ArgVal : ArgVals)
11452 Chains.push_back(ArgVal.getValue(1));
11453
11454 // Avoid emitting code for the store implementing the copy.
11455 const StoreInst *SI = ArgCopyIter->second.second;
11456 ElidedArgCopyInstrs.insert(SI);
11457
11458 // Check for uses of the argument again so that we can avoid exporting ArgVal
11459 // if it is't used by anything other than the store.
11460 for (const Value *U : Arg.users()) {
11461 if (U != SI) {
11462 ArgHasUses = true;
11463 break;
11464 }
11465 }
11466}
11467
11468void SelectionDAGISel::LowerArguments(const Function &F) {
11469 SelectionDAG &DAG = SDB->DAG;
11470 SDLoc dl = SDB->getCurSDLoc();
11471 const DataLayout &DL = DAG.getDataLayout();
11473
11474 // In Naked functions we aren't going to save any registers.
11475 if (F.hasFnAttribute(Attribute::Naked))
11476 return;
11477
11478 if (!FuncInfo->CanLowerReturn) {
11479 // Put in an sret pointer parameter before all the other parameters.
11480 SmallVector<EVT, 1> ValueVTs;
11482 PointerType::get(F.getContext(),
11484 ValueVTs);
11485
11486 // NOTE: Assuming that a pointer will never break down to more than one VT
11487 // or one register.
11489 Flags.setSRet();
11490 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
11491 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
11493 Ins.push_back(RetArg);
11494 }
11495
11496 // Look for stores of arguments to static allocas. Mark such arguments with a
11497 // flag to ask the target to give us the memory location of that argument if
11498 // available.
11499 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11501 ArgCopyElisionCandidates);
11502
11503 // Set up the incoming argument description vector.
11504 for (const Argument &Arg : F.args()) {
11505 unsigned ArgNo = Arg.getArgNo();
11506 SmallVector<EVT, 4> ValueVTs;
11507 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11508 bool isArgValueUsed = !Arg.use_empty();
11509 unsigned PartBase = 0;
11510 Type *FinalType = Arg.getType();
11511 if (Arg.hasAttribute(Attribute::ByVal))
11512 FinalType = Arg.getParamByValType();
11513 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11514 FinalType, F.getCallingConv(), F.isVarArg(), DL);
11515 for (unsigned Value = 0, NumValues = ValueVTs.size();
11516 Value != NumValues; ++Value) {
11517 EVT VT = ValueVTs[Value];
11518 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
11520
11521
11522 if (Arg.getType()->isPointerTy()) {
11523 Flags.setPointer();
11524 Flags.setPointerAddrSpace(
11525 cast<PointerType>(Arg.getType())->getAddressSpace());
11526 }
11527 if (Arg.hasAttribute(Attribute::ZExt))
11528 Flags.setZExt();
11529 if (Arg.hasAttribute(Attribute::SExt))
11530 Flags.setSExt();
11531 if (Arg.hasAttribute(Attribute::InReg)) {
11532 // If we are using vectorcall calling convention, a structure that is
11533 // passed InReg - is surely an HVA
11534 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11535 isa<StructType>(Arg.getType())) {
11536 // The first value of a structure is marked
11537 if (0 == Value)
11538 Flags.setHvaStart();
11539 Flags.setHva();
11540 }
11541 // Set InReg Flag
11542 Flags.setInReg();
11543 }
11544 if (Arg.hasAttribute(Attribute::StructRet))
11545 Flags.setSRet();
11546 if (Arg.hasAttribute(Attribute::SwiftSelf))
11547 Flags.setSwiftSelf();
11548 if (Arg.hasAttribute(Attribute::SwiftAsync))
11549 Flags.setSwiftAsync();
11550 if (Arg.hasAttribute(Attribute::SwiftError))
11551 Flags.setSwiftError();
11552 if (Arg.hasAttribute(Attribute::ByVal))
11553 Flags.setByVal();
11554 if (Arg.hasAttribute(Attribute::ByRef))
11555 Flags.setByRef();
11556 if (Arg.hasAttribute(Attribute::InAlloca)) {
11557 Flags.setInAlloca();
11558 // Set the byval flag for CCAssignFn callbacks that don't know about
11559 // inalloca. This way we can know how many bytes we should've allocated
11560 // and how many bytes a callee cleanup function will pop. If we port
11561 // inalloca to more targets, we'll have to add custom inalloca handling
11562 // in the various CC lowering callbacks.
11563 Flags.setByVal();
11564 }
11565 if (Arg.hasAttribute(Attribute::Preallocated)) {
11566 Flags.setPreallocated();
11567 // Set the byval flag for CCAssignFn callbacks that don't know about
11568 // preallocated. This way we can know how many bytes we should've
11569 // allocated and how many bytes a callee cleanup function will pop. If
11570 // we port preallocated to more targets, we'll have to add custom
11571 // preallocated handling in the various CC lowering callbacks.
11572 Flags.setByVal();
11573 }
11574
11575 // Certain targets (such as MIPS), may have a different ABI alignment
11576 // for a type depending on the context. Give the target a chance to
11577 // specify the alignment it wants.
11578 const Align OriginalAlignment(
11580 Flags.setOrigAlign(OriginalAlignment);
11581
11582 Align MemAlign;
11583 Type *ArgMemTy = nullptr;
11584 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11585 Flags.isByRef()) {
11586 if (!ArgMemTy)
11587 ArgMemTy = Arg.getPointeeInMemoryValueType();
11588
11589 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
11590
11591 // For in-memory arguments, size and alignment should be passed from FE.
11592 // BE will guess if this info is not there but there are cases it cannot
11593 // get right.
11594 if (auto ParamAlign = Arg.getParamStackAlign())
11595 MemAlign = *ParamAlign;
11596 else if ((ParamAlign = Arg.getParamAlign()))
11597 MemAlign = *ParamAlign;
11598 else
11599 MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
11600 if (Flags.isByRef())
11601 Flags.setByRefSize(MemSize);
11602 else
11603 Flags.setByValSize(MemSize);
11604 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
11605 MemAlign = *ParamAlign;
11606 } else {
11607 MemAlign = OriginalAlignment;
11608 }
11609 Flags.setMemAlign(MemAlign);
11610
11611 if (Arg.hasAttribute(Attribute::Nest))
11612 Flags.setNest();
11613 if (NeedsRegBlock)
11614 Flags.setInConsecutiveRegs();
11615 if (ArgCopyElisionCandidates.count(&Arg))
11616 Flags.setCopyElisionCandidate();
11617 if (Arg.hasAttribute(Attribute::Returned))
11618 Flags.setReturned();
11619
11621 *CurDAG->getContext(), F.getCallingConv(), VT);
11622 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
11623 *CurDAG->getContext(), F.getCallingConv(), VT);
11624 for (unsigned i = 0; i != NumRegs; ++i) {
11625 // For scalable vectors, use the minimum size; individual targets
11626 // are responsible for handling scalable vector arguments and
11627 // return values.
11628 ISD::InputArg MyFlags(
11629 Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
11630 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
11631 if (NumRegs > 1 && i == 0)
11632 MyFlags.Flags.setSplit();
11633 // if it isn't first piece, alignment must be 1
11634 else if (i > 0) {
11635 MyFlags.Flags.setOrigAlign(Align(1));
11636 if (i == NumRegs - 1)
11637 MyFlags.Flags.setSplitEnd();
11638 }
11639 Ins.push_back(MyFlags);
11640 }
11641 if (NeedsRegBlock && Value == NumValues - 1)
11642 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11643 PartBase += VT.getStoreSize().getKnownMinValue();
11644 }
11645 }
11646
11647 // Call the target to set up the argument values.
11649 SDValue NewRoot = TLI->LowerFormalArguments(
11650 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
11651
11652 // Verify that the target's LowerFormalArguments behaved as expected.
11653 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
11654 "LowerFormalArguments didn't return a valid chain!");
11655 assert(InVals.size() == Ins.size() &&
11656 "LowerFormalArguments didn't emit the correct number of values!");
11657 LLVM_DEBUG({
11658 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
11659 assert(InVals[i].getNode() &&
11660 "LowerFormalArguments emitted a null value!");
11661 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11662 "LowerFormalArguments emitted a value with the wrong type!");
11663 }
11664 });
11665
11666 // Update the DAG with the new chain value resulting from argument lowering.
11667 DAG.setRoot(NewRoot);
11668
11669 // Set up the argument values.
11670 unsigned i = 0;
11671 if (!FuncInfo->CanLowerReturn) {
11672 // Create a virtual register for the sret pointer, and put in a copy
11673 // from the sret argument into it.
11674 SmallVector<EVT, 1> ValueVTs;
11676 PointerType::get(F.getContext(),
11678 ValueVTs);
11679 MVT VT = ValueVTs[0].getSimpleVT();
11680 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
11681 std::optional<ISD::NodeType> AssertOp;
11682 SDValue ArgValue =
11683 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
11684 F.getCallingConv(), AssertOp);
11685
11686 MachineFunction& MF = SDB->DAG.getMachineFunction();
11688 Register SRetReg =
11689 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
11690 FuncInfo->DemoteRegister = SRetReg;
11691 NewRoot =
11692 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
11693 DAG.setRoot(NewRoot);
11694
11695 // i indexes lowered arguments. Bump it past the hidden sret argument.
11696 ++i;
11697 }
11698
11700 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
11701 for (const Argument &Arg : F.args()) {
11702 SmallVector<SDValue, 4> ArgValues;
11703 SmallVector<EVT, 4> ValueVTs;
11704 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11705 unsigned NumValues = ValueVTs.size();
11706 if (NumValues == 0)
11707 continue;
11708
11709 bool ArgHasUses = !Arg.use_empty();
11710
11711 // Elide the copying store if the target loaded this argument from a
11712 // suitable fixed stack object.
11713 if (Ins[i].Flags.isCopyElisionCandidate()) {
11714 unsigned NumParts = 0;
11715 for (EVT VT : ValueVTs)
11717 F.getCallingConv(), VT);
11718
11719 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
11720 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
11721 ArrayRef(&InVals[i], NumParts), ArgHasUses);
11722 }
11723
11724 // If this argument is unused then remember its value. It is used to generate
11725 // debugging information.
11726 bool isSwiftErrorArg =
11728 Arg.hasAttribute(Attribute::SwiftError);
11729 if (!ArgHasUses && !isSwiftErrorArg) {
11730 SDB->setUnusedArgValue(&Arg, InVals[i]);
11731
11732 // Also remember any frame index for use in FastISel.
11733 if (FrameIndexSDNode *FI =
11734 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
11735 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11736 }
11737
11738 for (unsigned Val = 0; Val != NumValues; ++Val) {
11739 EVT VT = ValueVTs[Val];
11741 F.getCallingConv(), VT);
11742 unsigned NumParts = TLI->getNumRegistersForCallingConv(
11743 *CurDAG->getContext(), F.getCallingConv(), VT);
11744
11745 // Even an apparent 'unused' swifterror argument needs to be returned. So
11746 // we do generate a copy for it that can be used on return from the
11747 // function.
11748 if (ArgHasUses || isSwiftErrorArg) {
11749 std::optional<ISD::NodeType> AssertOp;
11750 if (Arg.hasAttribute(Attribute::SExt))
11751 AssertOp = ISD::AssertSext;
11752 else if (Arg.hasAttribute(Attribute::ZExt))
11753 AssertOp = ISD::AssertZext;
11754
11755 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
11756 PartVT, VT, nullptr, NewRoot,
11757 F.getCallingConv(), AssertOp));
11758 }
11759
11760 i += NumParts;
11761 }
11762
11763 // We don't need to do anything else for unused arguments.
11764 if (ArgValues.empty())
11765 continue;
11766
11767 // Note down frame index.
11768 if (FrameIndexSDNode *FI =
11769 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
11770 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11771
11772 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
11773 SDB->getCurSDLoc());
11774
11775 SDB->setValue(&Arg, Res);
11777 // We want to associate the argument with the frame index, among
11778 // involved operands, that correspond to the lowest address. The
11779 // getCopyFromParts function, called earlier, is swapping the order of
11780 // the operands to BUILD_PAIR depending on endianness. The result of
11781 // that swapping is that the least significant bits of the argument will
11782 // be in the first operand of the BUILD_PAIR node, and the most
11783 // significant bits will be in the second operand.
11784 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
11785 if (LoadSDNode *LNode =
11786 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
11787 if (FrameIndexSDNode *FI =
11788 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
11789 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11790 }
11791
11792 // Analyses past this point are naive and don't expect an assertion.
11793 if (Res.getOpcode() == ISD::AssertZext)
11794 Res = Res.getOperand(0);
11795
11796 // Update the SwiftErrorVRegDefMap.
11797 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
11798 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11801 Reg);
11802 }
11803
11804 // If this argument is live outside of the entry block, insert a copy from
11805 // wherever we got it to the vreg that other BB's will reference it as.
11806 if (Res.getOpcode() == ISD::CopyFromReg) {
11807 // If we can, though, try to skip creating an unnecessary vreg.
11808 // FIXME: This isn't very clean... it would be nice to make this more
11809 // general.
11810 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11811 if (Register::isVirtualRegister(Reg)) {
11812 FuncInfo->ValueMap[&Arg] = Reg;
11813 continue;
11814 }
11815 }
11817 FuncInfo->InitializeRegForValue(&Arg);
11818 SDB->CopyToExportRegsIfNeeded(&Arg);
11819 }
11820 }
11821
11822 if (!Chains.empty()) {
11823 Chains.push_back(NewRoot);
11824 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
11825 }
11826
11827 DAG.setRoot(NewRoot);
11828
11829 assert(i == InVals.size() && "Argument register count mismatch!");
11830
11831 // If any argument copy elisions occurred and we have debug info, update the
11832 // stale frame indices used in the dbg.declare variable info table.
11833 if (!ArgCopyElisionFrameIndexMap.empty()) {
11836 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
11837 if (I != ArgCopyElisionFrameIndexMap.end())
11838 VI.updateStackSlot(I->second);
11839 }
11840 }
11841
11842 // Finally, if the target has anything special to do, allow it to do so.
11844}
11845
11846/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
11847/// ensure constants are generated when needed. Remember the virtual registers
11848/// that need to be added to the Machine PHI nodes as input. We cannot just
11849/// directly add them, because expansion might result in multiple MBB's for one
11850/// BB. As such, the start of the BB might correspond to a different MBB than
11851/// the end.
11852void
11853SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
11855
11857
11858 // Check PHI nodes in successors that expect a value to be available from this
11859 // block.
11860 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
11861 if (!isa<PHINode>(SuccBB->begin())) continue;
11862 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB);
11863
11864 // If this terminator has multiple identical successors (common for
11865 // switches), only handle each succ once.
11866 if (!SuccsHandled.insert(SuccMBB).second)
11867 continue;
11868
11870
11871 // At this point we know that there is a 1-1 correspondence between LLVM PHI
11872 // nodes and Machine PHI nodes, but the incoming operands have not been
11873 // emitted yet.
11874 for (const PHINode &PN : SuccBB->phis()) {
11875 // Ignore dead phi's.
11876 if (PN.use_empty())
11877 continue;
11878
11879 // Skip empty types
11880 if (PN.getType()->isEmptyTy())
11881 continue;
11882
11883 unsigned Reg;
11884 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
11885
11886 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
11887 unsigned &RegOut = ConstantsOut[C];
11888 if (RegOut == 0) {
11889 RegOut = FuncInfo.CreateRegs(C);
11890 // We need to zero/sign extend ConstantInt phi operands to match
11891 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
11892 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
11893 if (auto *CI = dyn_cast<ConstantInt>(C))
11894 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
11896 CopyValueToVirtualRegister(C, RegOut, ExtendType);
11897 }
11898 Reg = RegOut;
11899 } else {
11901 FuncInfo.ValueMap.find(PHIOp);
11902 if (I != FuncInfo.ValueMap.end())
11903 Reg = I->second;
11904 else {
11905 assert(isa<AllocaInst>(PHIOp) &&
11906 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
11907 "Didn't codegen value into a register!??");
11908 Reg = FuncInfo.CreateRegs(PHIOp);
11909 CopyValueToVirtualRegister(PHIOp, Reg);
11910 }
11911 }
11912
11913 // Remember that this register needs to added to the machine PHI node as
11914 // the input for this MBB.
11915 SmallVector<EVT, 4> ValueVTs;
11916 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
11917 for (EVT VT : ValueVTs) {
11918 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
11919 for (unsigned i = 0; i != NumRegisters; ++i)
11920 FuncInfo.PHINodesToUpdate.push_back(
11921 std::make_pair(&*MBBI++, Reg + i));
11922 Reg += NumRegisters;
11923 }
11924 }
11925 }
11926
11927 ConstantsOut.clear();
11928}
11929
11930MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
11932 if (++I == FuncInfo.MF->end())
11933 return nullptr;
11934 return &*I;
11935}
11936
11937/// During lowering new call nodes can be created (such as memset, etc.).
11938/// Those will become new roots of the current DAG, but complications arise
11939/// when they are tail calls. In such cases, the call lowering will update
11940/// the root, but the builder still needs to know that a tail call has been
11941/// lowered in order to avoid generating an additional return.
11942void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
11943 // If the node is null, we do have a tail call.
11944 if (MaybeTC.getNode() != nullptr)
11945 DAG.setRoot(MaybeTC);
11946 else
11947 HasTailCall = true;
11948}
11949
11950void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
11951 MachineBasicBlock *SwitchMBB,
11952 MachineBasicBlock *DefaultMBB) {
11953 MachineFunction *CurMF = FuncInfo.MF;
11954 MachineBasicBlock *NextMBB = nullptr;
11956 if (++BBI != FuncInfo.MF->end())
11957 NextMBB = &*BBI;
11958
11959 unsigned Size = W.LastCluster - W.FirstCluster + 1;
11960
11962
11963 if (Size == 2 && W.MBB == SwitchMBB) {
11964 // If any two of the cases has the same destination, and if one value
11965 // is the same as the other, but has one bit unset that the other has set,
11966 // use bit manipulation to do two compares at once. For example:
11967 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
11968 // TODO: This could be extended to merge any 2 cases in switches with 3
11969 // cases.
11970 // TODO: Handle cases where W.CaseBB != SwitchBB.
11971 CaseCluster &Small = *W.FirstCluster;
11972 CaseCluster &Big = *W.LastCluster;
11973
11974 if (Small.Low == Small.High && Big.Low == Big.High &&
11975 Small.MBB == Big.MBB) {
11976 const APInt &SmallValue = Small.Low->getValue();
11977 const APInt &BigValue = Big.Low->getValue();
11978
11979 // Check that there is only one bit different.
11980 APInt CommonBit = BigValue ^ SmallValue;
11981 if (CommonBit.isPowerOf2()) {
11982 SDValue CondLHS = getValue(Cond);
11983 EVT VT = CondLHS.getValueType();
11984 SDLoc DL = getCurSDLoc();
11985
11986 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
11987 DAG.getConstant(CommonBit, DL, VT));
11989 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
11990 ISD::SETEQ);
11991
11992 // Update successor info.
11993 // Both Small and Big will jump to Small.BB, so we sum up the
11994 // probabilities.
11995 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
11996 if (BPI)
11997 addSuccessorWithProb(
11998 SwitchMBB, DefaultMBB,
11999 // The default destination is the first successor in IR.
12000 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
12001 else
12002 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12003
12004 // Insert the true branch.
12005 SDValue BrCond =
12006 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
12007 DAG.getBasicBlock(Small.MBB));
12008 // Insert the false branch.
12009 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
12010 DAG.getBasicBlock(DefaultMBB));
12011
12012 DAG.setRoot(BrCond);
12013 return;
12014 }
12015 }
12016 }
12017
12018 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12019 // Here, we order cases by probability so the most likely case will be
12020 // checked first. However, two clusters can have the same probability in
12021 // which case their relative ordering is non-deterministic. So we use Low
12022 // as a tie-breaker as clusters are guaranteed to never overlap.
12023 llvm::sort(W.FirstCluster, W.LastCluster + 1,
12024 [](const CaseCluster &a, const CaseCluster &b) {
12025 return a.Prob != b.Prob ?
12026 a.Prob > b.Prob :
12027 a.Low->getValue().slt(b.Low->getValue());
12028 });
12029
12030 // Rearrange the case blocks so that the last one falls through if possible
12031 // without changing the order of probabilities.
12032 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12033 --I;
12034 if (I->Prob > W.LastCluster->Prob)
12035 break;
12036 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12037 std::swap(*I, *W.LastCluster);
12038 break;
12039 }
12040 }
12041 }
12042
12043 // Compute total probability.
12044 BranchProbability DefaultProb = W.DefaultProb;
12045 BranchProbability UnhandledProbs = DefaultProb;
12046 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12047 UnhandledProbs += I->Prob;
12048
12049 MachineBasicBlock *CurMBB = W.MBB;
12050 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12051 bool FallthroughUnreachable = false;
12052 MachineBasicBlock *Fallthrough;
12053 if (I == W.LastCluster) {
12054 // For the last cluster, fall through to the default destination.
12055 Fallthrough = DefaultMBB;
12056 FallthroughUnreachable = isa<UnreachableInst>(
12057 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
12058 } else {
12059 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
12060 CurMF->insert(BBI, Fallthrough);
12061 // Put Cond in a virtual register to make it available from the new blocks.
12063 }
12064 UnhandledProbs -= I->Prob;
12065
12066 switch (I->Kind) {
12067 case CC_JumpTable: {
12068 // FIXME: Optimize away range check based on pivot comparisons.
12069 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12070 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12071
12072 // The jump block hasn't been inserted yet; insert it here.
12073 MachineBasicBlock *JumpMBB = JT->MBB;
12074 CurMF->insert(BBI, JumpMBB);
12075
12076 auto JumpProb = I->Prob;
12077 auto FallthroughProb = UnhandledProbs;
12078
12079 // If the default statement is a target of the jump table, we evenly
12080 // distribute the default probability to successors of CurMBB. Also
12081 // update the probability on the edge from JumpMBB to Fallthrough.
12082 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12083 SE = JumpMBB->succ_end();
12084 SI != SE; ++SI) {
12085 if (*SI == DefaultMBB) {
12086 JumpProb += DefaultProb / 2;
12087 FallthroughProb -= DefaultProb / 2;
12088 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
12089 JumpMBB->normalizeSuccProbs();
12090 break;
12091 }
12092 }
12093
12094 // If the default clause is unreachable, propagate that knowledge into
12095 // JTH->FallthroughUnreachable which will use it to suppress the range
12096 // check.
12097 //
12098 // However, don't do this if we're doing branch target enforcement,
12099 // because a table branch _without_ a range check can be a tempting JOP
12100 // gadget - out-of-bounds inputs that are impossible in correct
12101 // execution become possible again if an attacker can influence the
12102 // control flow. So if an attacker doesn't already have a BTI bypass
12103 // available, we don't want them to be able to get one out of this
12104 // table branch.
12105 if (FallthroughUnreachable) {
12106 Function &CurFunc = CurMF->getFunction();
12107 if (!CurFunc.hasFnAttribute("branch-target-enforcement"))
12108 JTH->FallthroughUnreachable = true;
12109 }
12110
12111 if (!JTH->FallthroughUnreachable)
12112 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12113 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12114 CurMBB->normalizeSuccProbs();
12115
12116 // The jump table header will be inserted in our current block, do the
12117 // range check, and fall through to our fallthrough block.
12118 JTH->HeaderBB = CurMBB;
12119 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12120
12121 // If we're in the right place, emit the jump table header right now.
12122 if (CurMBB == SwitchMBB) {
12123 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
12124 JTH->Emitted = true;
12125 }
12126 break;
12127 }
12128 case CC_BitTests: {
12129 // FIXME: Optimize away range check based on pivot comparisons.
12130 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12131
12132 // The bit test blocks haven't been inserted yet; insert them here.
12133 for (BitTestCase &BTC : BTB->Cases)
12134 CurMF->insert(BBI, BTC.ThisBB);
12135
12136 // Fill in fields of the BitTestBlock.
12137 BTB->Parent = CurMBB;
12138 BTB->Default = Fallthrough;
12139
12140 BTB->DefaultProb = UnhandledProbs;
12141 // If the cases in bit test don't form a contiguous range, we evenly
12142 // distribute the probability on the edge to Fallthrough to two
12143 // successors of CurMBB.
12144 if (!BTB->ContiguousRange) {
12145 BTB->Prob += DefaultProb / 2;
12146 BTB->DefaultProb -= DefaultProb / 2;
12147 }
12148
12149 if (FallthroughUnreachable)
12150 BTB->FallthroughUnreachable = true;
12151
12152 // If we're in the right place, emit the bit test header right now.
12153 if (CurMBB == SwitchMBB) {
12154 visitBitTestHeader(*BTB, SwitchMBB);
12155 BTB->Emitted = true;
12156 }
12157 break;
12158 }
12159 case CC_Range: {
12160 const Value *RHS, *LHS, *MHS;
12162 if (I->Low == I->High) {
12163 // Check Cond == I->Low.
12164 CC = ISD::SETEQ;
12165 LHS = Cond;
12166 RHS=I->Low;
12167 MHS = nullptr;
12168 } else {
12169 // Check I->Low <= Cond <= I->High.
12170 CC = ISD::SETLE;
12171 LHS = I->Low;
12172 MHS = Cond;
12173 RHS = I->High;
12174 }
12175
12176 // If Fallthrough is unreachable, fold away the comparison.
12177 if (FallthroughUnreachable)
12178 CC = ISD::SETTRUE;
12179
12180 // The false probability is the sum of all unhandled cases.
12181 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12182 getCurSDLoc(), I->Prob, UnhandledProbs);
12183
12184 if (CurMBB == SwitchMBB)
12185 visitSwitchCase(CB, SwitchMBB);
12186 else
12187 SL->SwitchCases.push_back(CB);
12188
12189 break;
12190 }
12191 }
12192 CurMBB = Fallthrough;
12193 }
12194}
12195
12196void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12197 const SwitchWorkListItem &W,
12198 Value *Cond,
12199 MachineBasicBlock *SwitchMBB) {
12200 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12201 "Clusters not sorted?");
12202 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12203
12204 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12205 SL->computeSplitWorkItemInfo(W);
12206
12207 // Use the first element on the right as pivot since we will make less-than
12208 // comparisons against it.
12209 CaseClusterIt PivotCluster = FirstRight;
12210 assert(PivotCluster > W.FirstCluster);
12211 assert(PivotCluster <= W.LastCluster);
12212
12213 CaseClusterIt FirstLeft = W.FirstCluster;
12214 CaseClusterIt LastRight = W.LastCluster;
12215
12216 const ConstantInt *Pivot = PivotCluster->Low;
12217
12218 // New blocks will be inserted immediately after the current one.
12220 ++BBI;
12221
12222 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12223 // we can branch to its destination directly if it's squeezed exactly in
12224 // between the known lower bound and Pivot - 1.
12225 MachineBasicBlock *LeftMBB;
12226 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12227 FirstLeft->Low == W.GE &&
12228 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12229 LeftMBB = FirstLeft->MBB;
12230 } else {
12231 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12232 FuncInfo.MF->insert(BBI, LeftMBB);
12233 WorkList.push_back(
12234 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
12235 // Put Cond in a virtual register to make it available from the new blocks.
12237 }
12238
12239 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12240 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12241 // directly if RHS.High equals the current upper bound.
12242 MachineBasicBlock *RightMBB;
12243 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12244 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12245 RightMBB = FirstRight->MBB;
12246 } else {
12247 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12248 FuncInfo.MF->insert(BBI, RightMBB);
12249 WorkList.push_back(
12250 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
12251 // Put Cond in a virtual register to make it available from the new blocks.
12253 }
12254
12255 // Create the CaseBlock record that will be used to lower the branch.
12256 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12257 getCurSDLoc(), LeftProb, RightProb);
12258
12259 if (W.MBB == SwitchMBB)
12260 visitSwitchCase(CB, SwitchMBB);
12261 else
12262 SL->SwitchCases.push_back(CB);
12263}
12264
12265// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12266// from the swith statement.
12268 BranchProbability PeeledCaseProb) {
12269 if (PeeledCaseProb == BranchProbability::getOne())
12271 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12272
12273 uint32_t Numerator = CaseProb.getNumerator();
12274 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
12275 return BranchProbability(Numerator, std::max(Numerator, Denominator));
12276}
12277
12278// Try to peel the top probability case if it exceeds the threshold.
12279// Return current MachineBasicBlock for the switch statement if the peeling
12280// does not occur.
12281// If the peeling is performed, return the newly created MachineBasicBlock
12282// for the peeled switch statement. Also update Clusters to remove the peeled
12283// case. PeeledCaseProb is the BranchProbability for the peeled case.
12284MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12285 const SwitchInst &SI, CaseClusterVector &Clusters,
12286 BranchProbability &PeeledCaseProb) {
12287 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12288 // Don't perform if there is only one cluster or optimizing for size.
12289 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12291 SwitchMBB->getParent()->getFunction().hasMinSize())
12292 return SwitchMBB;
12293
12295 unsigned PeeledCaseIndex = 0;
12296 bool SwitchPeeled = false;
12297 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12298 CaseCluster &CC = Clusters[Index];
12299 if (CC.Prob < TopCaseProb)
12300 continue;
12301 TopCaseProb = CC.Prob;
12302 PeeledCaseIndex = Index;
12303 SwitchPeeled = true;
12304 }
12305 if (!SwitchPeeled)
12306 return SwitchMBB;
12307
12308 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12309 << TopCaseProb << "\n");
12310
12311 // Record the MBB for the peeled switch statement.
12312 MachineFunction::iterator BBI(SwitchMBB);
12313 ++BBI;
12314 MachineBasicBlock *PeeledSwitchMBB =
12316 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
12317
12318 ExportFromCurrentBlock(SI.getCondition());
12319 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12320 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
12321 nullptr, nullptr, TopCaseProb.getCompl()};
12322 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
12323
12324 Clusters.erase(PeeledCaseIt);
12325 for (CaseCluster &CC : Clusters) {
12326 LLVM_DEBUG(
12327 dbgs() << "Scale the probablity for one cluster, before scaling: "
12328 << CC.Prob << "\n");
12329 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
12330 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12331 }
12332 PeeledCaseProb = TopCaseProb;
12333 return PeeledSwitchMBB;
12334}
12335
12336void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12337 // Extract cases from the switch.
12339 CaseClusterVector Clusters;
12340 Clusters.reserve(SI.getNumCases());
12341 for (auto I : SI.cases()) {
12342 MachineBasicBlock *Succ = FuncInfo.getMBB(I.getCaseSuccessor());
12343 const ConstantInt *CaseVal = I.getCaseValue();
12344 BranchProbability Prob =
12345 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
12346 : BranchProbability(1, SI.getNumCases() + 1);
12347 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
12348 }
12349
12350 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(SI.getDefaultDest());
12351
12352 // Cluster adjacent cases with the same destination. We do this at all
12353 // optimization levels because it's cheap to do and will make codegen faster
12354 // if there are many clusters.
12355 sortAndRangeify(Clusters);
12356
12357 // The branch probablity of the peeled case.
12359 MachineBasicBlock *PeeledSwitchMBB =
12360 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12361
12362 // If there is only the default destination, jump there directly.
12363 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12364 if (Clusters.empty()) {
12365 assert(PeeledSwitchMBB == SwitchMBB);
12366 SwitchMBB->addSuccessor(DefaultMBB);
12367 if (DefaultMBB != NextBlock(SwitchMBB)) {
12368 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
12369 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
12370 }
12371 return;
12372 }
12373
12374 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
12375 DAG.getBFI());
12376 SL->findBitTestClusters(Clusters, &SI);
12377
12378 LLVM_DEBUG({
12379 dbgs() << "Case clusters: ";
12380 for (const CaseCluster &C : Clusters) {
12381 if (C.Kind == CC_JumpTable)
12382 dbgs() << "JT:";
12383 if (C.Kind == CC_BitTests)
12384 dbgs() << "BT:";
12385
12386 C.Low->getValue().print(dbgs(), true);
12387 if (C.Low != C.High) {
12388 dbgs() << '-';
12389 C.High->getValue().print(dbgs(), true);
12390 }
12391 dbgs() << ' ';
12392 }
12393 dbgs() << '\n';
12394 });
12395
12396 assert(!Clusters.empty());
12397 SwitchWorkList WorkList;
12398 CaseClusterIt First = Clusters.begin();
12399 CaseClusterIt Last = Clusters.end() - 1;
12400 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
12401 // Scale the branchprobability for DefaultMBB if the peel occurs and
12402 // DefaultMBB is not replaced.
12403 if (PeeledCaseProb != BranchProbability::getZero() &&
12404 DefaultMBB == FuncInfo.getMBB(SI.getDefaultDest()))
12405 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
12406 WorkList.push_back(
12407 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
12408
12409 while (!WorkList.empty()) {
12410 SwitchWorkListItem W = WorkList.pop_back_val();
12411 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12412
12413 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12414 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12415 // For optimized builds, lower large range as a balanced binary tree.
12416 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
12417 continue;
12418 }
12419
12420 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
12421 }
12422}
12423
12424void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12426 auto DL = getCurSDLoc();
12427 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12428 setValue(&I, DAG.getStepVector(DL, ResultVT));
12429}
12430
12431void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12433 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12434
12435 SDLoc DL = getCurSDLoc();
12436 SDValue V = getValue(I.getOperand(0));
12437 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12438
12439 if (VT.isScalableVector()) {
12441 return;
12442 }
12443
12444 // Use VECTOR_SHUFFLE for the fixed-length vector
12445 // to maintain existing behavior.
12447 unsigned NumElts = VT.getVectorMinNumElements();
12448 for (unsigned i = 0; i != NumElts; ++i)
12449 Mask.push_back(NumElts - 1 - i);
12450
12451 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
12452}
12453
12454void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
12455 auto DL = getCurSDLoc();
12456 SDValue InVec = getValue(I.getOperand(0));
12457 EVT OutVT =
12459
12460 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12461
12462 // ISD Node needs the input vectors split into two equal parts
12463 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12465 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12466 DAG.getVectorIdxConstant(OutNumElts, DL));
12467
12468 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12469 // legalisation and combines.
12470 if (OutVT.isFixedLengthVector()) {
12471 SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12472 createStrideMask(0, 2, OutNumElts));
12473 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12474 createStrideMask(1, 2, OutNumElts));
12475 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
12476 setValue(&I, Res);
12477 return;
12478 }
12479
12481 DAG.getVTList(OutVT, OutVT), Lo, Hi);
12482 setValue(&I, Res);
12483}
12484
12485void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
12486 auto DL = getCurSDLoc();
12487 EVT InVT = getValue(I.getOperand(0)).getValueType();
12488 SDValue InVec0 = getValue(I.getOperand(0));
12489 SDValue InVec1 = getValue(I.getOperand(1));
12491 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12492
12493 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12494 // legalisation and combines.
12495 if (OutVT.isFixedLengthVector()) {
12496 unsigned NumElts = InVT.getVectorMinNumElements();
12497 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
12498 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
12499 createInterleaveMask(NumElts, 2)));
12500 return;
12501 }
12502
12504 DAG.getVTList(InVT, InVT), InVec0, InVec1);
12505 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
12506 Res.getValue(1));
12507 setValue(&I, Res);
12508}
12509
12510void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12511 SmallVector<EVT, 4> ValueVTs;
12513 ValueVTs);
12514 unsigned NumValues = ValueVTs.size();
12515 if (NumValues == 0) return;
12516
12517 SmallVector<SDValue, 4> Values(NumValues);
12518 SDValue Op = getValue(I.getOperand(0));
12519
12520 for (unsigned i = 0; i != NumValues; ++i)
12521 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
12522 SDValue(Op.getNode(), Op.getResNo() + i));
12523
12525 DAG.getVTList(ValueVTs), Values));
12526}
12527
12528void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12530 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12531
12532 SDLoc DL = getCurSDLoc();
12533 SDValue V1 = getValue(I.getOperand(0));
12534 SDValue V2 = getValue(I.getOperand(1));
12535 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
12536
12537 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
12538 if (VT.isScalableVector()) {
12539 setValue(
12540 &I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
12542 Imm, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
12543 return;
12544 }
12545
12546 unsigned NumElts = VT.getVectorNumElements();
12547
12548 uint64_t Idx = (NumElts + Imm) % NumElts;
12549
12550 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12552 for (unsigned i = 0; i < NumElts; ++i)
12553 Mask.push_back(Idx + i);
12554 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
12555}
12556
12557// Consider the following MIR after SelectionDAG, which produces output in
12558// phyregs in the first case or virtregs in the second case.
12559//
12560// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12561// %5:gr32 = COPY $ebx
12562// %6:gr32 = COPY $edx
12563// %1:gr32 = COPY %6:gr32
12564// %0:gr32 = COPY %5:gr32
12565//
12566// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12567// %1:gr32 = COPY %6:gr32
12568// %0:gr32 = COPY %5:gr32
12569//
12570// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12571// Given %1, we'd like to return $edx in the first case and %6 in the second.
12572//
12573// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12574// to a single virtreg (such as %0). The remaining outputs monotonically
12575// increase in virtreg number from there. If a callbr has no outputs, then it
12576// should not have a corresponding callbr landingpad; in fact, the callbr
12577// landingpad would not even be able to refer to such a callbr.
12579 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
12580 // There is definitely at least one copy.
12581 assert(MI->getOpcode() == TargetOpcode::COPY &&
12582 "start of copy chain MUST be COPY");
12583 Reg = MI->getOperand(1).getReg();
12584 MI = MRI.def_begin(Reg)->getParent();
12585 // There may be an optional second copy.
12586 if (MI->getOpcode() == TargetOpcode::COPY) {
12587 assert(Reg.isVirtual() && "expected COPY of virtual register");
12588 Reg = MI->getOperand(1).getReg();
12589 assert(Reg.isPhysical() && "expected COPY of physical register");
12590 MI = MRI.def_begin(Reg)->getParent();
12591 }
12592 // The start of the chain must be an INLINEASM_BR.
12593 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12594 "end of copy chain MUST be INLINEASM_BR");
12595 return Reg;
12596}
12597
12598// We must do this walk rather than the simpler
12599// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
12600// otherwise we will end up with copies of virtregs only valid along direct
12601// edges.
12602void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
12603 SmallVector<EVT, 8> ResultVTs;
12604 SmallVector<SDValue, 8> ResultValues;
12605 const auto *CBR =
12606 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
12607
12611
12612 unsigned InitialDef = FuncInfo.ValueMap[CBR];
12613 SDValue Chain = DAG.getRoot();
12614
12615 // Re-parse the asm constraints string.
12616 TargetLowering::AsmOperandInfoVector TargetConstraints =
12617 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
12618 for (auto &T : TargetConstraints) {
12619 SDISelAsmOperandInfo OpInfo(T);
12620 if (OpInfo.Type != InlineAsm::isOutput)
12621 continue;
12622
12623 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
12624 // individual constraint.
12625 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
12626
12627 switch (OpInfo.ConstraintType) {
12630 // Fill in OpInfo.AssignedRegs.Regs.
12631 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
12632
12633 // getRegistersForValue may produce 1 to many registers based on whether
12634 // the OpInfo.ConstraintVT is legal on the target or not.
12635 for (unsigned &Reg : OpInfo.AssignedRegs.Regs) {
12636 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
12637 if (Register::isPhysicalRegister(OriginalDef))
12638 FuncInfo.MBB->addLiveIn(OriginalDef);
12639 // Update the assigned registers to use the original defs.
12640 Reg = OriginalDef;
12641 }
12642
12643 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
12644 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
12645 ResultValues.push_back(V);
12646 ResultVTs.push_back(OpInfo.ConstraintVT);
12647 break;
12648 }
12650 SDValue Flag;
12651 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
12652 OpInfo, DAG);
12653 ++InitialDef;
12654 ResultValues.push_back(V);
12655 ResultVTs.push_back(OpInfo.ConstraintVT);
12656 break;
12657 }
12658 default:
12659 break;
12660 }
12661 }
12663 DAG.getVTList(ResultVTs), ResultValues);
12664 setValue(&I, V);
12665}
unsigned const MachineRegisterInfo * MRI
@ Poison
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition: FastISel.cpp:944
#define Check(C,...)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::optional< ConstantRange > getRange(Value *V, const InstrInfoQuery &IIQ)
Helper method to get range from metadata or attribute.
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file contains the declarations for metadata subclasses.
#define T1
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Module.h This file contains the declarations for the Module class.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< unsigned, TypeSize > > &Regs, const SDValue &N)
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, ISD::MemIndexType &IndexType, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static void findWasmUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const unsigned char *MatcherTable, unsigned &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:78
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1010
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
an instruction to allocate memory on the stack
Definition: Instructions.h:61
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:122
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:115
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition: Function.cpp:368
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:49
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:495
This class represents the atomic memcpy intrinsic i.e.
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:696
@ Add
*p = old + v
Definition: Instructions.h:712
@ FAdd
*p = old + v
Definition: Instructions.h:733
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:726
@ Or
*p = old | v
Definition: Instructions.h:720
@ Sub
*p = old - v
Definition: Instructions.h:714
@ And
*p = old & v
Definition: Instructions.h:716
@ Xor
*p = old ^ v
Definition: Instructions.h:722
@ FSub
*p = old - v
Definition: Instructions.h:736
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:748
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:724
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:730
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:744
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:728
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:740
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:752
@ Nand
*p = ~(old & v)
Definition: Instructions.h:718
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:367
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
Definition: BasicBlock.cpp:571
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:219
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:386
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:239
const Instruction & back() const
Definition: BasicBlock.h:473
This class represents a no-op cast from one type to another.
bool test(unsigned Idx) const
Definition: BitVector.h:461
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
BitVector & set()
Definition: BitVector.h:351
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
The address of a basic block.
Definition: Constants.h:890
Conditional or Unconditional Branch instruction.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static uint32_t getDenominator()
static BranchProbability getOne()
static BranchProbability getUnknown()
uint32_t getNumerator() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2143
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1523
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1385
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2119
Value * getCalledOperand() const
Definition: InstrTypes.h:1458
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1410
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1391
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2027
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1323
unsigned arg_size() const
Definition: InstrTypes.h:1408
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1542
bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:747
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:757
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition: Constants.h:584
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1097
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2295
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:269
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:850
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:206
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:857
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:155
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:146
A signed pointer, in the ptrauth sense.
Definition: Constants.h:1012
This class represents a range of values.
Definition: ConstantRange.h:47
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition: Constants.h:508
This is an important base class in LLVM.
Definition: Constant.h:42
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
unsigned getNonMetadataArgCount() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Debug location.
Base class for variables.
std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:695
bool isBigEndian() const
Definition: DataLayout.h:196
unsigned getAllocaAddrSpace() const
Definition: DataLayout.h:233
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
Definition: DataLayout.h:377
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:429
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:842
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
iterator_range< location_op_iterator > getValues() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
A debug info location.
Definition: DebugLoc.h:33
DILocation * getInlinedAt() const
Definition: DebugLoc.cpp:39
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:211
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:311
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition: TypeSize.h:317
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:322
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
bool allowReassoc() const
Flag queries.
Definition: FMF.h:65
An instruction for ordering other memory operations.
Definition: Instructions.h:420
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:680
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
Register CreateRegs(const Value *V)
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
MachineBasicBlock * getMBB(const BasicBlock *BB) const
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
BitVector DescribedArgs
Bitvector with a bit set if corresponding argument is described in ArgDbgValues.
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
Register InitializeRegForValue(const Value *V)
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineBasicBlock * MBB
MBB - The current block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
unsigned getCurrentCallSite()
Get the call site currently being processed, if any. Return zero if none.
void setCurrentCallSite(unsigned Site)
Set the call site currently being processed.
MachineRegisterInfo * RegInfo
Register CreateReg(MVT VT, bool isDivergent=false)
CreateReg - Allocate a single virtual register for the given type.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
Register getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:142
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
Type * getReturnType() const
Definition: DerivedTypes.h:124
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition: Function.h:807
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:249
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:702
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:349
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:281
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1993
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:357
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:254
size_t arg_size() const
Definition: Function.h:899
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition: Function.h:232
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:743
Garbage collection metadata for a single function.
Definition: GCMetadata.h:78
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Definition: GCMetadata.h:118
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:915
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:567
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
This instruction inserts a struct field of array element value into an aggregate value.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:466
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1713
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Definition: DerivedTypes.h:51
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:55
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:174
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:346
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:236
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
Metadata node.
Definition: Metadata.h:1069
Machine Value Type.
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void setIsEHCatchretTarget(bool V=true)
Indicates if this is a target block of a catchret.
void setIsCleanupFuncletEntry(bool V=true)
Indicates if this is the entry block of a cleanup funclet.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasPatchPoint(bool s=true)
void setHasStackMap(bool s=true)
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
Description of the location of a variable whose Address is valid and unchanging during function execu...
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
void setCallsUnwindInit(bool b)
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void setHasEHCatchret(bool V)
void setCallsEHReturn(bool b)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
auto getInStackSlotVariableDbgInfo()
Returns the collection of variables for which we have debug info and that have been assigned a stack ...
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
ArrayRef< std::pair< MCRegister, Register > > liveins() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool contains(const KeyT &Key) const
Definition: MapVector.h:163
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition: MapVector.h:118
Representation for a specific memory location.
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:176
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition: Operator.h:77
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1852
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed.
Definition: Operator.h:152
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
void CopyValueToVirtualRegister(const Value *V, unsigned Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, unsigned Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const BranchInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void init(GCFunctionInfo *gfi, AAResults *AA, AssumptionCache *AC, const TargetLibraryInfo *li)
DenseMap< const Constant *, unsigned > ConstantsOut
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineFunction * MF
virtual void emitFunctionEntryCode()
SwiftErrorValueTracking * SwiftError
std::unique_ptr< SelectionDAGBuilder > SDB
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
Help to insert SDNodeFlags automatically in transforming.
Definition: SelectionDAG.h:364
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:226
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:736
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:968
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:567
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:489
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
BlockFrequencyInfo * getBFI() const
Definition: SelectionDAG.h:503
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
void addMMRAMetadata(const SDNode *Node, MDNode *MMRA)
Set MMRAMetadata to be associated with Node.
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:493
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:451
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:842
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:487
ProfileSummaryInfo * getPSI() const
Definition: SelectionDAG.h:502
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:741
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:495
SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:876
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:488
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:787
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:722
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:690
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:482
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:813
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
Definition: SelectionDAG.h:499
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:500
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:576
void addPCSections(const SDNode *Node, MDNode *MD)
Set PCSections to be associated with Node.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:570
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:892
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:346
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:367
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:502
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:981
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
void clear()
Clear the memory usage of this object.
An instruction for storing to memory.
Definition: Instructions.h:290
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:600
Class to represent struct types.
Definition: DerivedTypes.h:216
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Multiway switch.
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
TargetIntrinsicInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *) const
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC, bool ZeroIsPoison, const ConstantRange *VScaleRange) const
Return the minimum number of bits required to hold the maximum possible number of trailing zero vecto...
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
std::vector< ArgListEntry > ArgListTy
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual bool supportPtrAuthBundles() const
Return true if the target supports ptrauth operand bundles.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual MVT getJumpTableRegTy(const DataLayout &DL) const
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const
Return a TargetTransformInfo for a given function.
TargetOptions Options
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
unsigned getID() const
Return the register class ID number.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
bool hasConditionalLoadStoreForType(Type *Ty=nullptr) const
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:373
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:766
bool isWasm() const
Tests whether the target is wasm (32- and 64-bit).
Definition: Triple.h:1021
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
Definition: Triple.h:911
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:345
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:261
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:251
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:224
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:221
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition: Type.h:212
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:343
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1833
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
Base class of all SIMD vector types.
Definition: DerivedTypes.h:403
Type * getElementType() const
Definition: DerivedTypes.h:436
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:202
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
const ParentTy * getParent() const
Definition: ilist_node.h:32
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:86
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition: CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
Definition: CallingConv.h:245
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
Definition: CallingConv.h:163
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:243
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1194
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1190
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:752
@ CONVERGENCECTRL_ANCHOR
Definition: ISDOpcodes.h:1460
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:490
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1344
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:1066
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1407
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1440
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:153
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1337
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:573
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:743
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:374
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1223
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1339
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1309
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1340
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1070
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:246
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:380
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1089
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:813
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:497
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:205
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:157
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1322
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:840
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:557
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1425
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1429
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:716
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1301
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1093
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1439
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:491
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
Definition: ISDOpcodes.h:494
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1335
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:953
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:236
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1336
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1267
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:996
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:387
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1342
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1170
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:141
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1176
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:935
@ CONVERGENCECTRL_GLUE
Definition: ISDOpcodes.h:1466
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:804
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
Definition: ISDOpcodes.h:1228
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1256
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:107
@ CONVERGENCECTRL_ENTRY
Definition: ISDOpcodes.h:1461
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:1115
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1422
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:751
@ WRITE_REGISTER
Definition: ISDOpcodes.h:125
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1289
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1426
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1343
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:334
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1338
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
Definition: ISDOpcodes.h:1231
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:1120
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1124
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
Definition: ISDOpcodes.h:600
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:514
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:521
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:356
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:756
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1305
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1441
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:229
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:641
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1219
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1345
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:215
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:330
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1293
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1434
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:930
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:1185
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1084
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1061
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:734
@ PtrAuthGlobalAddress
A ptrauth constant.
Definition: ISDOpcodes.h:90
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1333
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:587
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:124
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:549
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:810
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1279
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:906
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1397
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1341
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:120
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:1028
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1283
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:366
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:338
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1109
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:696
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition: ISDOpcodes.h:605
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:393
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:938
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:765
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1442
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
Definition: ISDOpcodes.h:1242
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
Definition: ISDOpcodes.h:1165
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition: ISDOpcodes.h:135
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:100
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1347
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1331
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:479
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:1047
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1332
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:886
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1250
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:708
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1276
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:190
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition: ISDOpcodes.h:704
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1423
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:407
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition: ISDOpcodes.h:223
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:538
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:626
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1330
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:1001
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:919
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition: ISDOpcodes.h:668
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition: ISDOpcodes.h:112
@ CONVERGENCECTRL_LOOP
Definition: ISDOpcodes.h:1462
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1162
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:905
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1430
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:147
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:816
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1138
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1408
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:1181
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1346
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:507
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:347
@ AssertZext
Definition: ISDOpcodes.h:62
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
Definition: ISDOpcodes.h:594
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1388
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
Definition: ISDOpcodes.h:1052
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1273
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:198
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:529
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1565
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1603
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:875
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
VScaleVal_match m_VScale()
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1589
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Reg
All possible values of the reg field in the ModR/M byte.
@ ReallyHidden
Definition: CommandLine.h:138
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:463
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:147
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebStrict
This corresponds to "fpexcept.strict".
Definition: FPEnv.h:41
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition: FPEnv.h:40
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
constexpr float log2ef
Definition: MathExtras.h:66
constexpr double e
Definition: MathExtras.h:47
constexpr float ln2f
Definition: MathExtras.h:64
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:353
@ Offset
Definition: DWP.cpp:480
@ Length
Definition: DWP.cpp:480
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1722
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1680
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:255
bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
@ Done
Definition: Threading.h:61
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2098
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:555
gep_type_iterator gep_type_end(const User *GEP)
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition: STLExtras.h:876
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1647
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition: STLExtras.h:322
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition: Analysis.cpp:199
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition: Local.cpp:2545
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2242
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:535
DWARFExpression::Operation Op
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition: Analysis.cpp:221
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2082
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, const LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2070
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
unsigned succ_size(const MachineBasicBlock *BB)
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:33
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition: bit.h:327
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
#define NC
Definition: regutils.h:42
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:281
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:381
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:74
uint64_t getScalarStoreSize() const
Definition: ValueTypes.h:388
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:275
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:291
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:147
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:341
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:359
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:350
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:371
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:275
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:307
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:367
bool isFixedLengthVector() const
Definition: ValueTypes.h:178
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:168
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:314
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:283
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:204
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:174
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:319
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:157
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:102
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:327
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:439
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:152
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition: InlineAsm.h:126
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:237
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< unsigned, 4 > Regs
This list holds the registers assigned to the values.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< std::pair< unsigned, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
bool hasAllowReassociation() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
A cluster of case labels.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)