LLVM 22.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
57#include "llvm/IR/Argument.h"
58#include "llvm/IR/Attributes.h"
59#include "llvm/IR/BasicBlock.h"
60#include "llvm/IR/CFG.h"
61#include "llvm/IR/CallingConv.h"
62#include "llvm/IR/Constant.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/DataLayout.h"
66#include "llvm/IR/DebugInfo.h"
71#include "llvm/IR/Function.h"
73#include "llvm/IR/InlineAsm.h"
74#include "llvm/IR/InstrTypes.h"
77#include "llvm/IR/Intrinsics.h"
78#include "llvm/IR/IntrinsicsAArch64.h"
79#include "llvm/IR/IntrinsicsAMDGPU.h"
80#include "llvm/IR/IntrinsicsWebAssembly.h"
81#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/Metadata.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Operator.h"
87#include "llvm/IR/Statepoint.h"
88#include "llvm/IR/Type.h"
89#include "llvm/IR/User.h"
90#include "llvm/IR/Value.h"
91#include "llvm/MC/MCContext.h"
96#include "llvm/Support/Debug.h"
104#include <cstddef>
105#include <limits>
106#include <optional>
107#include <tuple>
108
109using namespace llvm;
110using namespace PatternMatch;
111using namespace SwitchCG;
112
113#define DEBUG_TYPE "isel"
114
115/// LimitFloatPrecision - Generate low-precision inline sequences for
116/// some float libcalls (6, 8 or 12 bits).
117static unsigned LimitFloatPrecision;
118
119static cl::opt<bool>
120 InsertAssertAlign("insert-assert-align", cl::init(true),
121 cl::desc("Insert the experimental `assertalign` node."),
123
125 LimitFPPrecision("limit-float-precision",
126 cl::desc("Generate low-precision inline sequences "
127 "for some float libcalls"),
129 cl::init(0));
130
132 "switch-peel-threshold", cl::Hidden, cl::init(66),
133 cl::desc("Set the case probability threshold for peeling the case from a "
134 "switch statement. A value greater than 100 will void this "
135 "optimization"));
136
137// Limit the width of DAG chains. This is important in general to prevent
138// DAG-based analysis from blowing up. For example, alias analysis and
139// load clustering may not complete in reasonable time. It is difficult to
140// recognize and avoid this situation within each individual analysis, and
141// future analyses are likely to have the same behavior. Limiting DAG width is
142// the safe approach and will be especially important with global DAGs.
143//
144// MaxParallelChains default is arbitrarily high to avoid affecting
145// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
146// sequence over this should have been converted to llvm.memcpy by the
147// frontend. It is easy to induce this behavior with .ll code such as:
148// %buffer = alloca [4096 x i8]
149// %data = load [4096 x i8]* %argPtr
150// store [4096 x i8] %data, [4096 x i8]* %buffer
151static const unsigned MaxParallelChains = 64;
152
154 const SDValue *Parts, unsigned NumParts,
155 MVT PartVT, EVT ValueVT, const Value *V,
156 SDValue InChain,
157 std::optional<CallingConv::ID> CC);
158
159/// getCopyFromParts - Create a value that contains the specified legal parts
160/// combined into the value they represent. If the parts combine to a type
161/// larger than ValueVT then AssertOp can be used to specify whether the extra
162/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
163/// (ISD::AssertSext).
164static SDValue
165getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
166 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
167 SDValue InChain,
168 std::optional<CallingConv::ID> CC = std::nullopt,
169 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
170 // Let the target assemble the parts if it wants to
171 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
172 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
173 PartVT, ValueVT, CC))
174 return Val;
175
176 if (ValueVT.isVector())
177 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
178 InChain, CC);
179
180 assert(NumParts > 0 && "No parts to assemble!");
181 SDValue Val = Parts[0];
182
183 if (NumParts > 1) {
184 // Assemble the value from multiple parts.
185 if (ValueVT.isInteger()) {
186 unsigned PartBits = PartVT.getSizeInBits();
187 unsigned ValueBits = ValueVT.getSizeInBits();
188
189 // Assemble the power of 2 part.
190 unsigned RoundParts = llvm::bit_floor(NumParts);
191 unsigned RoundBits = PartBits * RoundParts;
192 EVT RoundVT = RoundBits == ValueBits ?
193 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
194 SDValue Lo, Hi;
195
196 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
197
198 if (RoundParts > 2) {
199 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
200 InChain);
201 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
202 PartVT, HalfVT, V, InChain);
203 } else {
204 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
205 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
206 }
207
208 if (DAG.getDataLayout().isBigEndian())
209 std::swap(Lo, Hi);
210
211 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
212
213 if (RoundParts < NumParts) {
214 // Assemble the trailing non-power-of-2 part.
215 unsigned OddParts = NumParts - RoundParts;
216 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
217 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
218 OddVT, V, InChain, CC);
219
220 // Combine the round and odd parts.
221 Lo = Val;
222 if (DAG.getDataLayout().isBigEndian())
223 std::swap(Lo, Hi);
224 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
225 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
226 Hi = DAG.getNode(
227 ISD::SHL, DL, TotalVT, Hi,
228 DAG.getShiftAmountConstant(Lo.getValueSizeInBits(), TotalVT, DL));
229 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
230 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
231 }
232 } else if (PartVT.isFloatingPoint()) {
233 // FP split into multiple FP parts (for ppcf128)
234 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
235 "Unexpected split");
236 SDValue Lo, Hi;
237 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
238 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
239 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
240 std::swap(Lo, Hi);
241 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
242 } else {
243 // FP split into integer parts (soft fp)
244 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
245 !PartVT.isVector() && "Unexpected split");
246 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
247 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
248 InChain, CC);
249 }
250 }
251
252 // There is now one part, held in Val. Correct it to match ValueVT.
253 // PartEVT is the type of the register class that holds the value.
254 // ValueVT is the type of the inline asm operation.
255 EVT PartEVT = Val.getValueType();
256
257 if (PartEVT == ValueVT)
258 return Val;
259
260 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
261 ValueVT.bitsLT(PartEVT)) {
262 // For an FP value in an integer part, we need to truncate to the right
263 // width first.
264 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
265 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
266 }
267
268 // Handle types that have the same size.
269 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
270 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
271
272 // Handle types with different sizes.
273 if (PartEVT.isInteger() && ValueVT.isInteger()) {
274 if (ValueVT.bitsLT(PartEVT)) {
275 // For a truncate, see if we have any information to
276 // indicate whether the truncated bits will always be
277 // zero or sign-extension.
278 if (AssertOp)
279 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
280 DAG.getValueType(ValueVT));
281 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
282 }
283 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
284 }
285
286 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
287 // FP_ROUND's are always exact here.
288 if (ValueVT.bitsLT(Val.getValueType())) {
289
290 SDValue NoChange =
292
293 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
294 llvm::Attribute::StrictFP)) {
295 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
296 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
297 NoChange);
298 }
299
300 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
301 }
302
303 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
304 }
305
306 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
307 // then truncating.
308 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
309 ValueVT.bitsLT(PartEVT)) {
310 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
311 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
312 }
313
314 report_fatal_error("Unknown mismatch in getCopyFromParts!");
315}
316
318 const Twine &ErrMsg) {
320 if (!I)
321 return Ctx.emitError(ErrMsg);
322
323 if (const CallInst *CI = dyn_cast<CallInst>(I))
324 if (CI->isInlineAsm()) {
325 return Ctx.diagnose(DiagnosticInfoInlineAsm(
326 *CI, ErrMsg + ", possible invalid constraint for vector type"));
327 }
328
329 return Ctx.emitError(I, ErrMsg);
330}
331
332/// getCopyFromPartsVector - Create a value that contains the specified legal
333/// parts combined into the value they represent. If the parts combine to a
334/// type larger than ValueVT then AssertOp can be used to specify whether the
335/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
336/// ValueVT (ISD::AssertSext).
338 const SDValue *Parts, unsigned NumParts,
339 MVT PartVT, EVT ValueVT, const Value *V,
340 SDValue InChain,
341 std::optional<CallingConv::ID> CallConv) {
342 assert(ValueVT.isVector() && "Not a vector value");
343 assert(NumParts > 0 && "No parts to assemble!");
344 const bool IsABIRegCopy = CallConv.has_value();
345
346 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
347 SDValue Val = Parts[0];
348
349 // Handle a multi-element vector.
350 if (NumParts > 1) {
351 EVT IntermediateVT;
352 MVT RegisterVT;
353 unsigned NumIntermediates;
354 unsigned NumRegs;
355
356 if (IsABIRegCopy) {
358 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
359 NumIntermediates, RegisterVT);
360 } else {
361 NumRegs =
362 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
363 NumIntermediates, RegisterVT);
364 }
365
366 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
367 NumParts = NumRegs; // Silence a compiler warning.
368 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
369 assert(RegisterVT.getSizeInBits() ==
370 Parts[0].getSimpleValueType().getSizeInBits() &&
371 "Part type sizes don't match!");
372
373 // Assemble the parts into intermediate operands.
374 SmallVector<SDValue, 8> Ops(NumIntermediates);
375 if (NumIntermediates == NumParts) {
376 // If the register was not expanded, truncate or copy the value,
377 // as appropriate.
378 for (unsigned i = 0; i != NumParts; ++i)
379 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
380 V, InChain, CallConv);
381 } else if (NumParts > 0) {
382 // If the intermediate type was expanded, build the intermediate
383 // operands from the parts.
384 assert(NumParts % NumIntermediates == 0 &&
385 "Must expand into a divisible number of parts!");
386 unsigned Factor = NumParts / NumIntermediates;
387 for (unsigned i = 0; i != NumIntermediates; ++i)
388 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
389 IntermediateVT, V, InChain, CallConv);
390 }
391
392 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
393 // intermediate operands.
394 EVT BuiltVectorTy =
395 IntermediateVT.isVector()
397 *DAG.getContext(), IntermediateVT.getScalarType(),
398 IntermediateVT.getVectorElementCount() * NumParts)
400 IntermediateVT.getScalarType(),
401 NumIntermediates);
402 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
404 DL, BuiltVectorTy, Ops);
405 }
406
407 // There is now one part, held in Val. Correct it to match ValueVT.
408 EVT PartEVT = Val.getValueType();
409
410 if (PartEVT == ValueVT)
411 return Val;
412
413 if (PartEVT.isVector()) {
414 // Vector/Vector bitcast.
415 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
416 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
417
418 // If the parts vector has more elements than the value vector, then we
419 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
420 // Extract the elements we want.
421 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
424 (PartEVT.getVectorElementCount().isScalable() ==
425 ValueVT.getVectorElementCount().isScalable()) &&
426 "Cannot narrow, it would be a lossy transformation");
427 PartEVT =
429 ValueVT.getVectorElementCount());
430 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
431 DAG.getVectorIdxConstant(0, DL));
432 if (PartEVT == ValueVT)
433 return Val;
434 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
435 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
436
437 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
438 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
439 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
440 }
441
442 // Promoted vector extract
443 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
444 }
445
446 // Trivial bitcast if the types are the same size and the destination
447 // vector type is legal.
448 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
449 TLI.isTypeLegal(ValueVT))
450 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
451
452 if (ValueVT.getVectorNumElements() != 1) {
453 // Certain ABIs require that vectors are passed as integers. For vectors
454 // are the same size, this is an obvious bitcast.
455 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
456 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
457 } else if (ValueVT.bitsLT(PartEVT)) {
458 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
459 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
460 // Drop the extra bits.
461 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
462 return DAG.getBitcast(ValueVT, Val);
463 }
464
466 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
467 return DAG.getUNDEF(ValueVT);
468 }
469
470 // Handle cases such as i8 -> <1 x i1>
471 EVT ValueSVT = ValueVT.getVectorElementType();
472 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
473 unsigned ValueSize = ValueSVT.getSizeInBits();
474 if (ValueSize == PartEVT.getSizeInBits()) {
475 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
476 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
477 // It's possible a scalar floating point type gets softened to integer and
478 // then promoted to a larger integer. If PartEVT is the larger integer
479 // we need to truncate it and then bitcast to the FP type.
480 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
481 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
482 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
483 Val = DAG.getBitcast(ValueSVT, Val);
484 } else {
485 Val = ValueVT.isFloatingPoint()
486 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
487 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
488 }
489 }
490
491 return DAG.getBuildVector(ValueVT, DL, Val);
492}
493
494static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
495 SDValue Val, SDValue *Parts, unsigned NumParts,
496 MVT PartVT, const Value *V,
497 std::optional<CallingConv::ID> CallConv);
498
499/// getCopyToParts - Create a series of nodes that contain the specified value
500/// split into legal parts. If the parts contain more bits than Val, then, for
501/// integers, ExtendKind can be used to specify how to generate the extra bits.
502static void
504 unsigned NumParts, MVT PartVT, const Value *V,
505 std::optional<CallingConv::ID> CallConv = std::nullopt,
506 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
507 // Let the target split the parts if it wants to
508 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
509 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
510 CallConv))
511 return;
512 EVT ValueVT = Val.getValueType();
513
514 // Handle the vector case separately.
515 if (ValueVT.isVector())
516 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
517 CallConv);
518
519 unsigned OrigNumParts = NumParts;
521 "Copying to an illegal type!");
522
523 if (NumParts == 0)
524 return;
525
526 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
527 EVT PartEVT = PartVT;
528 if (PartEVT == ValueVT) {
529 assert(NumParts == 1 && "No-op copy with multiple parts!");
530 Parts[0] = Val;
531 return;
532 }
533
534 unsigned PartBits = PartVT.getSizeInBits();
535 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
536 // If the parts cover more bits than the value has, promote the value.
537 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
538 assert(NumParts == 1 && "Do not know what to promote to!");
539 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
540 } else {
541 if (ValueVT.isFloatingPoint()) {
542 // FP values need to be bitcast, then extended if they are being put
543 // into a larger container.
544 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
545 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
546 }
547 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
548 ValueVT.isInteger() &&
549 "Unknown mismatch!");
550 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
551 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
552 if (PartVT == MVT::x86mmx)
553 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
554 }
555 } else if (PartBits == ValueVT.getSizeInBits()) {
556 // Different types of the same size.
557 assert(NumParts == 1 && PartEVT != ValueVT);
558 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
559 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
560 // If the parts cover less bits than value has, truncate the value.
561 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
562 ValueVT.isInteger() &&
563 "Unknown mismatch!");
564 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
565 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
566 if (PartVT == MVT::x86mmx)
567 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
568 }
569
570 // The value may have changed - recompute ValueVT.
571 ValueVT = Val.getValueType();
572 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
573 "Failed to tile the value with PartVT!");
574
575 if (NumParts == 1) {
576 if (PartEVT != ValueVT) {
578 "scalar-to-vector conversion failed");
579 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
580 }
581
582 Parts[0] = Val;
583 return;
584 }
585
586 // Expand the value into multiple parts.
587 if (NumParts & (NumParts - 1)) {
588 // The number of parts is not a power of 2. Split off and copy the tail.
589 assert(PartVT.isInteger() && ValueVT.isInteger() &&
590 "Do not know what to expand to!");
591 unsigned RoundParts = llvm::bit_floor(NumParts);
592 unsigned RoundBits = RoundParts * PartBits;
593 unsigned OddParts = NumParts - RoundParts;
594 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
595 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
596
597 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
598 CallConv);
599
600 if (DAG.getDataLayout().isBigEndian())
601 // The odd parts were reversed by getCopyToParts - unreverse them.
602 std::reverse(Parts + RoundParts, Parts + NumParts);
603
604 NumParts = RoundParts;
605 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
606 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
607 }
608
609 // The number of parts is a power of 2. Repeatedly bisect the value using
610 // EXTRACT_ELEMENT.
611 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
613 ValueVT.getSizeInBits()),
614 Val);
615
616 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
617 for (unsigned i = 0; i < NumParts; i += StepSize) {
618 unsigned ThisBits = StepSize * PartBits / 2;
619 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
620 SDValue &Part0 = Parts[i];
621 SDValue &Part1 = Parts[i+StepSize/2];
622
623 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
624 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
625 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
626 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
627
628 if (ThisBits == PartBits && ThisVT != PartVT) {
629 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
630 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
631 }
632 }
633 }
634
635 if (DAG.getDataLayout().isBigEndian())
636 std::reverse(Parts, Parts + OrigNumParts);
637}
638
640 const SDLoc &DL, EVT PartVT) {
641 if (!PartVT.isVector())
642 return SDValue();
643
644 EVT ValueVT = Val.getValueType();
645 EVT PartEVT = PartVT.getVectorElementType();
646 EVT ValueEVT = ValueVT.getVectorElementType();
647 ElementCount PartNumElts = PartVT.getVectorElementCount();
648 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
649
650 // We only support widening vectors with equivalent element types and
651 // fixed/scalable properties. If a target needs to widen a fixed-length type
652 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
653 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
654 PartNumElts.isScalable() != ValueNumElts.isScalable())
655 return SDValue();
656
657 // Have a try for bf16 because some targets share its ABI with fp16.
658 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
660 "Cannot widen to illegal type");
661 Val = DAG.getNode(ISD::BITCAST, DL,
662 ValueVT.changeVectorElementType(MVT::f16), Val);
663 } else if (PartEVT != ValueEVT) {
664 return SDValue();
665 }
666
667 // Widening a scalable vector to another scalable vector is done by inserting
668 // the vector into a larger undef one.
669 if (PartNumElts.isScalable())
670 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
671 Val, DAG.getVectorIdxConstant(0, DL));
672
673 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
674 // undef elements.
676 DAG.ExtractVectorElements(Val, Ops);
677 SDValue EltUndef = DAG.getUNDEF(PartEVT);
678 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
679
680 // FIXME: Use CONCAT for 2x -> 4x.
681 return DAG.getBuildVector(PartVT, DL, Ops);
682}
683
684/// getCopyToPartsVector - Create a series of nodes that contain the specified
685/// value split into legal parts.
686static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
687 SDValue Val, SDValue *Parts, unsigned NumParts,
688 MVT PartVT, const Value *V,
689 std::optional<CallingConv::ID> CallConv) {
690 EVT ValueVT = Val.getValueType();
691 assert(ValueVT.isVector() && "Not a vector");
692 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
693 const bool IsABIRegCopy = CallConv.has_value();
694
695 if (NumParts == 1) {
696 EVT PartEVT = PartVT;
697 if (PartEVT == ValueVT) {
698 // Nothing to do.
699 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
700 // Bitconvert vector->vector case.
701 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
702 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
703 Val = Widened;
704 } else if (PartVT.isVector() &&
706 ValueVT.getVectorElementType()) &&
707 PartEVT.getVectorElementCount() ==
708 ValueVT.getVectorElementCount()) {
709
710 // Promoted vector extract
711 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
712 } else if (PartEVT.isVector() &&
713 PartEVT.getVectorElementType() !=
714 ValueVT.getVectorElementType() &&
715 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
717 // Combination of widening and promotion.
718 EVT WidenVT =
720 PartVT.getVectorElementCount());
721 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
722 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
723 } else {
724 // Don't extract an integer from a float vector. This can happen if the
725 // FP type gets softened to integer and then promoted. The promotion
726 // prevents it from being picked up by the earlier bitcast case.
727 if (ValueVT.getVectorElementCount().isScalar() &&
728 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
729 // If we reach this condition and PartVT is FP, this means that
730 // ValueVT is also FP and both have a different size, otherwise we
731 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
732 // would be invalid since that would mean the smaller FP type has to
733 // be extended to the larger one.
734 if (PartVT.isFloatingPoint()) {
735 Val = DAG.getBitcast(ValueVT.getScalarType(), Val);
736 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
737 } else
738 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
739 DAG.getVectorIdxConstant(0, DL));
740 } else {
741 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
742 assert(PartVT.getFixedSizeInBits() > ValueSize &&
743 "lossy conversion of vector to scalar type");
744 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
745 Val = DAG.getBitcast(IntermediateType, Val);
746 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
747 }
748 }
749
750 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
751 Parts[0] = Val;
752 return;
753 }
754
755 // Handle a multi-element vector.
756 EVT IntermediateVT;
757 MVT RegisterVT;
758 unsigned NumIntermediates;
759 unsigned NumRegs;
760 if (IsABIRegCopy) {
762 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
763 RegisterVT);
764 } else {
765 NumRegs =
766 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
767 NumIntermediates, RegisterVT);
768 }
769
770 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
771 NumParts = NumRegs; // Silence a compiler warning.
772 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
773
774 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
775 "Mixing scalable and fixed vectors when copying in parts");
776
777 std::optional<ElementCount> DestEltCnt;
778
779 if (IntermediateVT.isVector())
780 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
781 else
782 DestEltCnt = ElementCount::getFixed(NumIntermediates);
783
784 EVT BuiltVectorTy = EVT::getVectorVT(
785 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
786
787 if (ValueVT == BuiltVectorTy) {
788 // Nothing to do.
789 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
790 // Bitconvert vector->vector case.
791 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
792 } else {
793 if (BuiltVectorTy.getVectorElementType().bitsGT(
794 ValueVT.getVectorElementType())) {
795 // Integer promotion.
796 ValueVT = EVT::getVectorVT(*DAG.getContext(),
797 BuiltVectorTy.getVectorElementType(),
798 ValueVT.getVectorElementCount());
799 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
800 }
801
802 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
803 Val = Widened;
804 }
805 }
806
807 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
808
809 // Split the vector into intermediate operands.
810 SmallVector<SDValue, 8> Ops(NumIntermediates);
811 for (unsigned i = 0; i != NumIntermediates; ++i) {
812 if (IntermediateVT.isVector()) {
813 // This does something sensible for scalable vectors - see the
814 // definition of EXTRACT_SUBVECTOR for further details.
815 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
816 Ops[i] =
817 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
818 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
819 } else {
820 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
821 DAG.getVectorIdxConstant(i, DL));
822 }
823 }
824
825 // Split the intermediate operands into legal parts.
826 if (NumParts == NumIntermediates) {
827 // If the register was not expanded, promote or copy the value,
828 // as appropriate.
829 for (unsigned i = 0; i != NumParts; ++i)
830 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
831 } else if (NumParts > 0) {
832 // If the intermediate type was expanded, split each the value into
833 // legal parts.
834 assert(NumIntermediates != 0 && "division by zero");
835 assert(NumParts % NumIntermediates == 0 &&
836 "Must expand into a divisible number of parts!");
837 unsigned Factor = NumParts / NumIntermediates;
838 for (unsigned i = 0; i != NumIntermediates; ++i)
839 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
840 CallConv);
841 }
842}
843
844static void failForInvalidBundles(const CallBase &I, StringRef Name,
845 ArrayRef<uint32_t> AllowedBundles) {
846 if (I.hasOperandBundlesOtherThan(AllowedBundles)) {
847 ListSeparator LS;
848 std::string Error;
850 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
851 OperandBundleUse U = I.getOperandBundleAt(i);
852 if (!is_contained(AllowedBundles, U.getTagID()))
853 OS << LS << U.getTagName();
854 }
856 Twine("cannot lower ", Name)
857 .concat(Twine(" with arbitrary operand bundles: ", Error)));
858 }
859}
860
862 EVT valuevt, std::optional<CallingConv::ID> CC)
863 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
864 RegCount(1, regs.size()), CallConv(CC) {}
865
867 const DataLayout &DL, Register Reg, Type *Ty,
868 std::optional<CallingConv::ID> CC) {
869 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
870
871 CallConv = CC;
872
873 for (EVT ValueVT : ValueVTs) {
874 unsigned NumRegs =
876 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
877 : TLI.getNumRegisters(Context, ValueVT);
878 MVT RegisterVT =
880 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
881 : TLI.getRegisterType(Context, ValueVT);
882 for (unsigned i = 0; i != NumRegs; ++i)
883 Regs.push_back(Reg + i);
884 RegVTs.push_back(RegisterVT);
885 RegCount.push_back(NumRegs);
886 Reg = Reg.id() + NumRegs;
887 }
888}
889
891 FunctionLoweringInfo &FuncInfo,
892 const SDLoc &dl, SDValue &Chain,
893 SDValue *Glue, const Value *V) const {
894 // A Value with type {} or [0 x %t] needs no registers.
895 if (ValueVTs.empty())
896 return SDValue();
897
898 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
899
900 // Assemble the legal parts into the final values.
901 SmallVector<SDValue, 4> Values(ValueVTs.size());
903 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
904 // Copy the legal parts from the registers.
905 EVT ValueVT = ValueVTs[Value];
906 unsigned NumRegs = RegCount[Value];
907 MVT RegisterVT = isABIMangled()
909 *DAG.getContext(), *CallConv, RegVTs[Value])
910 : RegVTs[Value];
911
912 Parts.resize(NumRegs);
913 for (unsigned i = 0; i != NumRegs; ++i) {
914 SDValue P;
915 if (!Glue) {
916 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
917 } else {
918 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
919 *Glue = P.getValue(2);
920 }
921
922 Chain = P.getValue(1);
923 Parts[i] = P;
924
925 // If the source register was virtual and if we know something about it,
926 // add an assert node.
927 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
928 continue;
929
931 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
932 if (!LOI)
933 continue;
934
935 unsigned RegSize = RegisterVT.getScalarSizeInBits();
936 unsigned NumSignBits = LOI->NumSignBits;
937 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
938
939 if (NumZeroBits == RegSize) {
940 // The current value is a zero.
941 // Explicitly express that as it would be easier for
942 // optimizations to kick in.
943 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
944 continue;
945 }
946
947 // FIXME: We capture more information than the dag can represent. For
948 // now, just use the tightest assertzext/assertsext possible.
949 bool isSExt;
950 EVT FromVT(MVT::Other);
951 if (NumZeroBits) {
952 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
953 isSExt = false;
954 } else if (NumSignBits > 1) {
955 FromVT =
956 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
957 isSExt = true;
958 } else {
959 continue;
960 }
961 // Add an assertion node.
962 assert(FromVT != MVT::Other);
963 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
964 RegisterVT, P, DAG.getValueType(FromVT));
965 }
966
967 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
968 RegisterVT, ValueVT, V, Chain, CallConv);
969 Part += NumRegs;
970 Parts.clear();
971 }
972
973 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
974}
975
977 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
978 const Value *V,
979 ISD::NodeType PreferredExtendType) const {
980 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
981 ISD::NodeType ExtendKind = PreferredExtendType;
982
983 // Get the list of the values's legal parts.
984 unsigned NumRegs = Regs.size();
985 SmallVector<SDValue, 8> Parts(NumRegs);
986 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
987 unsigned NumParts = RegCount[Value];
988
989 MVT RegisterVT = isABIMangled()
991 *DAG.getContext(), *CallConv, RegVTs[Value])
992 : RegVTs[Value];
993
994 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
995 ExtendKind = ISD::ZERO_EXTEND;
996
997 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
998 NumParts, RegisterVT, V, CallConv, ExtendKind);
999 Part += NumParts;
1000 }
1001
1002 // Copy the parts into the registers.
1003 SmallVector<SDValue, 8> Chains(NumRegs);
1004 for (unsigned i = 0; i != NumRegs; ++i) {
1005 SDValue Part;
1006 if (!Glue) {
1007 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
1008 } else {
1009 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
1010 *Glue = Part.getValue(1);
1011 }
1012
1013 Chains[i] = Part.getValue(0);
1014 }
1015
1016 if (NumRegs == 1 || Glue)
1017 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1018 // flagged to it. That is the CopyToReg nodes and the user are considered
1019 // a single scheduling unit. If we create a TokenFactor and return it as
1020 // chain, then the TokenFactor is both a predecessor (operand) of the
1021 // user as well as a successor (the TF operands are flagged to the user).
1022 // c1, f1 = CopyToReg
1023 // c2, f2 = CopyToReg
1024 // c3 = TokenFactor c1, c2
1025 // ...
1026 // = op c3, ..., f2
1027 Chain = Chains[NumRegs-1];
1028 else
1029 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1030}
1031
1033 unsigned MatchingIdx, const SDLoc &dl,
1034 SelectionDAG &DAG,
1035 std::vector<SDValue> &Ops) const {
1036 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1037
1038 InlineAsm::Flag Flag(Code, Regs.size());
1039 if (HasMatching)
1040 Flag.setMatchingOp(MatchingIdx);
1041 else if (!Regs.empty() && Regs.front().isVirtual()) {
1042 // Put the register class of the virtual registers in the flag word. That
1043 // way, later passes can recompute register class constraints for inline
1044 // assembly as well as normal instructions.
1045 // Don't do this for tied operands that can use the regclass information
1046 // from the def.
1048 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1049 Flag.setRegClass(RC->getID());
1050 }
1051
1052 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1053 Ops.push_back(Res);
1054
1055 if (Code == InlineAsm::Kind::Clobber) {
1056 // Clobbers should always have a 1:1 mapping with registers, and may
1057 // reference registers that have illegal (e.g. vector) types. Hence, we
1058 // shouldn't try to apply any sort of splitting logic to them.
1059 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1060 "No 1:1 mapping from clobbers to regs?");
1062 (void)SP;
1063 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1064 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1065 assert(
1066 (Regs[I] != SP ||
1068 "If we clobbered the stack pointer, MFI should know about it.");
1069 }
1070 return;
1071 }
1072
1073 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1074 MVT RegisterVT = RegVTs[Value];
1075 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1076 RegisterVT);
1077 for (unsigned i = 0; i != NumRegs; ++i) {
1078 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1079 Register TheReg = Regs[Reg++];
1080 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1081 }
1082 }
1083}
1084
1088 unsigned I = 0;
1089 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1090 unsigned RegCount = std::get<0>(CountAndVT);
1091 MVT RegisterVT = std::get<1>(CountAndVT);
1092 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1093 for (unsigned E = I + RegCount; I != E; ++I)
1094 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1095 }
1096 return OutVec;
1097}
1098
1100 AssumptionCache *ac,
1101 const TargetLibraryInfo *li) {
1102 BatchAA = aa;
1103 AC = ac;
1104 GFI = gfi;
1105 LibInfo = li;
1106 Context = DAG.getContext();
1107 LPadToCallSiteMap.clear();
1108 SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1109 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1110 *DAG.getMachineFunction().getFunction().getParent());
1111}
1112
1114 NodeMap.clear();
1115 UnusedArgNodeMap.clear();
1116 PendingLoads.clear();
1117 PendingExports.clear();
1118 PendingConstrainedFP.clear();
1119 PendingConstrainedFPStrict.clear();
1120 CurInst = nullptr;
1121 HasTailCall = false;
1122 SDNodeOrder = LowestSDNodeOrder;
1123 StatepointLowering.clear();
1124}
1125
1127 DanglingDebugInfoMap.clear();
1128}
1129
1130// Update DAG root to include dependencies on Pending chains.
1131SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1132 SDValue Root = DAG.getRoot();
1133
1134 if (Pending.empty())
1135 return Root;
1136
1137 // Add current root to PendingChains, unless we already indirectly
1138 // depend on it.
1139 if (Root.getOpcode() != ISD::EntryToken) {
1140 unsigned i = 0, e = Pending.size();
1141 for (; i != e; ++i) {
1142 assert(Pending[i].getNode()->getNumOperands() > 1);
1143 if (Pending[i].getNode()->getOperand(0) == Root)
1144 break; // Don't add the root if we already indirectly depend on it.
1145 }
1146
1147 if (i == e)
1148 Pending.push_back(Root);
1149 }
1150
1151 if (Pending.size() == 1)
1152 Root = Pending[0];
1153 else
1154 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1155
1156 DAG.setRoot(Root);
1157 Pending.clear();
1158 return Root;
1159}
1160
1164
1166 // Chain up all pending constrained intrinsics together with all
1167 // pending loads, by simply appending them to PendingLoads and
1168 // then calling getMemoryRoot().
1169 PendingLoads.reserve(PendingLoads.size() +
1170 PendingConstrainedFP.size() +
1171 PendingConstrainedFPStrict.size());
1172 PendingLoads.append(PendingConstrainedFP.begin(),
1173 PendingConstrainedFP.end());
1174 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1175 PendingConstrainedFPStrict.end());
1176 PendingConstrainedFP.clear();
1177 PendingConstrainedFPStrict.clear();
1178 return getMemoryRoot();
1179}
1180
1182 // We need to emit pending fpexcept.strict constrained intrinsics,
1183 // so append them to the PendingExports list.
1184 PendingExports.append(PendingConstrainedFPStrict.begin(),
1185 PendingConstrainedFPStrict.end());
1186 PendingConstrainedFPStrict.clear();
1187 return updateRoot(PendingExports);
1188}
1189
1191 DILocalVariable *Variable,
1193 DebugLoc DL) {
1194 assert(Variable && "Missing variable");
1195
1196 // Check if address has undef value.
1197 if (!Address || isa<UndefValue>(Address) ||
1198 (Address->use_empty() && !isa<Argument>(Address))) {
1199 LLVM_DEBUG(
1200 dbgs()
1201 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1202 return;
1203 }
1204
1205 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1206
1207 SDValue &N = NodeMap[Address];
1208 if (!N.getNode() && isa<Argument>(Address))
1209 // Check unused arguments map.
1210 N = UnusedArgNodeMap[Address];
1211 SDDbgValue *SDV;
1212 if (N.getNode()) {
1213 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1214 Address = BCI->getOperand(0);
1215 // Parameters are handled specially.
1216 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1217 if (IsParameter && FINode) {
1218 // Byval parameter. We have a frame index at this point.
1219 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1220 /*IsIndirect*/ true, DL, SDNodeOrder);
1221 } else if (isa<Argument>(Address)) {
1222 // Address is an argument, so try to emit its dbg value using
1223 // virtual register info from the FuncInfo.ValueMap.
1224 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1225 FuncArgumentDbgValueKind::Declare, N);
1226 return;
1227 } else {
1228 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1229 true, DL, SDNodeOrder);
1230 }
1231 DAG.AddDbgValue(SDV, IsParameter);
1232 } else {
1233 // If Address is an argument then try to emit its dbg value using
1234 // virtual register info from the FuncInfo.ValueMap.
1235 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1236 FuncArgumentDbgValueKind::Declare, N)) {
1237 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1238 << " (could not emit func-arg dbg_value)\n");
1239 }
1240 }
1241}
1242
1244 // Add SDDbgValue nodes for any var locs here. Do so before updating
1245 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1246 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1247 // Add SDDbgValue nodes for any var locs here. Do so before updating
1248 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1249 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1250 It != End; ++It) {
1251 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1252 dropDanglingDebugInfo(Var, It->Expr);
1253 if (It->Values.isKillLocation(It->Expr)) {
1254 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1255 continue;
1256 }
1257 SmallVector<Value *> Values(It->Values.location_ops());
1258 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1259 It->Values.hasArgList())) {
1260 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1262 FnVarLocs->getDILocalVariable(It->VariableID),
1263 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1264 }
1265 }
1266 }
1267
1268 // We must skip DbgVariableRecords if they've already been processed above as
1269 // we have just emitted the debug values resulting from assignment tracking
1270 // analysis, making any existing DbgVariableRecords redundant (and probably
1271 // less correct). We still need to process DbgLabelRecords. This does sink
1272 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1273 // be important as it does so deterministcally and ordering between
1274 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1275 // printing).
1276 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1277 // Is there is any debug-info attached to this instruction, in the form of
1278 // DbgRecord non-instruction debug-info records.
1279 for (DbgRecord &DR : I.getDbgRecordRange()) {
1280 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1281 assert(DLR->getLabel() && "Missing label");
1282 SDDbgLabel *SDV =
1283 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1284 DAG.AddDbgLabel(SDV);
1285 continue;
1286 }
1287
1288 if (SkipDbgVariableRecords)
1289 continue;
1291 DILocalVariable *Variable = DVR.getVariable();
1294
1296 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1297 continue;
1298 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1299 << "\n");
1301 DVR.getDebugLoc());
1302 continue;
1303 }
1304
1305 // A DbgVariableRecord with no locations is a kill location.
1307 if (Values.empty()) {
1309 SDNodeOrder);
1310 continue;
1311 }
1312
1313 // A DbgVariableRecord with an undef or absent location is also a kill
1314 // location.
1315 if (llvm::any_of(Values,
1316 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1318 SDNodeOrder);
1319 continue;
1320 }
1321
1322 bool IsVariadic = DVR.hasArgList();
1323 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1324 SDNodeOrder, IsVariadic)) {
1325 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1326 DVR.getDebugLoc(), SDNodeOrder);
1327 }
1328 }
1329}
1330
1332 visitDbgInfo(I);
1333
1334 // Set up outgoing PHI node register values before emitting the terminator.
1335 if (I.isTerminator()) {
1336 HandlePHINodesInSuccessorBlocks(I.getParent());
1337 }
1338
1339 ++SDNodeOrder;
1340 CurInst = &I;
1341
1342 // Set inserted listener only if required.
1343 bool NodeInserted = false;
1344 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1345 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1346 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);
1347 if (PCSectionsMD || MMRA) {
1348 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1349 DAG, [&](SDNode *) { NodeInserted = true; });
1350 }
1351
1352 visit(I.getOpcode(), I);
1353
1354 if (!I.isTerminator() && !HasTailCall &&
1355 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1357
1358 // Handle metadata.
1359 if (PCSectionsMD || MMRA) {
1360 auto It = NodeMap.find(&I);
1361 if (It != NodeMap.end()) {
1362 if (PCSectionsMD)
1363 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1364 if (MMRA)
1365 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1366 } else if (NodeInserted) {
1367 // This should not happen; if it does, don't let it go unnoticed so we can
1368 // fix it. Relevant visit*() function is probably missing a setValue().
1369 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1370 << I.getModule()->getName() << "]\n";
1371 LLVM_DEBUG(I.dump());
1372 assert(false);
1373 }
1374 }
1375
1376 CurInst = nullptr;
1377}
1378
1379void SelectionDAGBuilder::visitPHI(const PHINode &) {
1380 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1381}
1382
1383void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1384 // Note: this doesn't use InstVisitor, because it has to work with
1385 // ConstantExpr's in addition to instructions.
1386 switch (Opcode) {
1387 default: llvm_unreachable("Unknown instruction type encountered!");
1388 // Build the switch statement using the Instruction.def file.
1389#define HANDLE_INST(NUM, OPCODE, CLASS) \
1390 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1391#include "llvm/IR/Instruction.def"
1392 }
1393}
1394
1396 DILocalVariable *Variable,
1397 DebugLoc DL, unsigned Order,
1400 // For variadic dbg_values we will now insert poison.
1401 // FIXME: We can potentially recover these!
1403 for (const Value *V : Values) {
1404 auto *Poison = PoisonValue::get(V->getType());
1406 }
1407 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1408 /*IsIndirect=*/false, DL, Order,
1409 /*IsVariadic=*/true);
1410 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1411 return true;
1412}
1413
1415 DILocalVariable *Var,
1416 DIExpression *Expr,
1417 bool IsVariadic, DebugLoc DL,
1418 unsigned Order) {
1419 if (IsVariadic) {
1420 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1421 return;
1422 }
1423 // TODO: Dangling debug info will eventually either be resolved or produce
1424 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1425 // between the original dbg.value location and its resolved DBG_VALUE,
1426 // which we should ideally fill with an extra poison DBG_VALUE.
1427 assert(Values.size() == 1);
1428 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1429}
1430
1432 const DIExpression *Expr) {
1433 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1434 DIVariable *DanglingVariable = DDI.getVariable();
1435 DIExpression *DanglingExpr = DDI.getExpression();
1436 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1437 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1438 << printDDI(nullptr, DDI) << "\n");
1439 return true;
1440 }
1441 return false;
1442 };
1443
1444 for (auto &DDIMI : DanglingDebugInfoMap) {
1445 DanglingDebugInfoVector &DDIV = DDIMI.second;
1446
1447 // If debug info is to be dropped, run it through final checks to see
1448 // whether it can be salvaged.
1449 for (auto &DDI : DDIV)
1450 if (isMatchingDbgValue(DDI))
1451 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1452
1453 erase_if(DDIV, isMatchingDbgValue);
1454 }
1455}
1456
1457// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1458// generate the debug data structures now that we've seen its definition.
1460 SDValue Val) {
1461 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1462 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1463 return;
1464
1465 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1466 for (auto &DDI : DDIV) {
1467 DebugLoc DL = DDI.getDebugLoc();
1468 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1469 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1470 DILocalVariable *Variable = DDI.getVariable();
1471 DIExpression *Expr = DDI.getExpression();
1472 assert(Variable->isValidLocationForIntrinsic(DL) &&
1473 "Expected inlined-at fields to agree");
1474 SDDbgValue *SDV;
1475 if (Val.getNode()) {
1476 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1477 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1478 // we couldn't resolve it directly when examining the DbgValue intrinsic
1479 // in the first place we should not be more successful here). Unless we
1480 // have some test case that prove this to be correct we should avoid
1481 // calling EmitFuncArgumentDbgValue here.
1482 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1483 FuncArgumentDbgValueKind::Value, Val)) {
1484 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1485 << printDDI(V, DDI) << "\n");
1486 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1487 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1488 // inserted after the definition of Val when emitting the instructions
1489 // after ISel. An alternative could be to teach
1490 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1491 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1492 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1493 << ValSDNodeOrder << "\n");
1494 SDV = getDbgValue(Val, Variable, Expr, DL,
1495 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1496 DAG.AddDbgValue(SDV, false);
1497 } else
1498 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1499 << printDDI(V, DDI)
1500 << " in EmitFuncArgumentDbgValue\n");
1501 } else {
1502 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1503 << "\n");
1504 auto Poison = PoisonValue::get(V->getType());
1505 auto SDV =
1506 DAG.getConstantDbgValue(Variable, Expr, Poison, DL, DbgSDNodeOrder);
1507 DAG.AddDbgValue(SDV, false);
1508 }
1509 }
1510 DDIV.clear();
1511}
1512
1514 DanglingDebugInfo &DDI) {
1515 // TODO: For the variadic implementation, instead of only checking the fail
1516 // state of `handleDebugValue`, we need know specifically which values were
1517 // invalid, so that we attempt to salvage only those values when processing
1518 // a DIArgList.
1519 const Value *OrigV = V;
1520 DILocalVariable *Var = DDI.getVariable();
1521 DIExpression *Expr = DDI.getExpression();
1522 DebugLoc DL = DDI.getDebugLoc();
1523 unsigned SDOrder = DDI.getSDNodeOrder();
1524
1525 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1526 // that DW_OP_stack_value is desired.
1527 bool StackValue = true;
1528
1529 // Can this Value can be encoded without any further work?
1530 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1531 return;
1532
1533 // Attempt to salvage back through as many instructions as possible. Bail if
1534 // a non-instruction is seen, such as a constant expression or global
1535 // variable. FIXME: Further work could recover those too.
1536 while (isa<Instruction>(V)) {
1537 const Instruction &VAsInst = *cast<const Instruction>(V);
1538 // Temporary "0", awaiting real implementation.
1540 SmallVector<Value *, 4> AdditionalValues;
1541 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1542 Expr->getNumLocationOperands(), Ops,
1543 AdditionalValues);
1544 // If we cannot salvage any further, and haven't yet found a suitable debug
1545 // expression, bail out.
1546 if (!V)
1547 break;
1548
1549 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1550 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1551 // here for variadic dbg_values, remove that condition.
1552 if (!AdditionalValues.empty())
1553 break;
1554
1555 // New value and expr now represent this debuginfo.
1556 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1557
1558 // Some kind of simplification occurred: check whether the operand of the
1559 // salvaged debug expression can be encoded in this DAG.
1560 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1561 LLVM_DEBUG(
1562 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1563 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1564 return;
1565 }
1566 }
1567
1568 // This was the final opportunity to salvage this debug information, and it
1569 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1570 // any earlier variable location.
1571 assert(OrigV && "V shouldn't be null");
1572 auto *Poison = PoisonValue::get(OrigV->getType());
1573 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Poison, DL, SDNodeOrder);
1574 DAG.AddDbgValue(SDV, false);
1575 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1576 << printDDI(OrigV, DDI) << "\n");
1577}
1578
1580 DIExpression *Expr,
1581 DebugLoc DbgLoc,
1582 unsigned Order) {
1586 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1587 /*IsVariadic*/ false);
1588}
1589
1591 DILocalVariable *Var,
1592 DIExpression *Expr, DebugLoc DbgLoc,
1593 unsigned Order, bool IsVariadic) {
1594 if (Values.empty())
1595 return true;
1596
1597 // Filter EntryValue locations out early.
1598 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1599 return true;
1600
1601 SmallVector<SDDbgOperand> LocationOps;
1602 SmallVector<SDNode *> Dependencies;
1603 for (const Value *V : Values) {
1604 // Constant value.
1607 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1608 continue;
1609 }
1610
1611 // Look through IntToPtr constants.
1612 if (auto *CE = dyn_cast<ConstantExpr>(V))
1613 if (CE->getOpcode() == Instruction::IntToPtr) {
1614 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1615 continue;
1616 }
1617
1618 // If the Value is a frame index, we can create a FrameIndex debug value
1619 // without relying on the DAG at all.
1620 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1621 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1622 if (SI != FuncInfo.StaticAllocaMap.end()) {
1623 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1624 continue;
1625 }
1626 }
1627
1628 // Do not use getValue() in here; we don't want to generate code at
1629 // this point if it hasn't been done yet.
1630 SDValue N = NodeMap[V];
1631 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1632 N = UnusedArgNodeMap[V];
1633
1634 if (N.getNode()) {
1635 // Only emit func arg dbg value for non-variadic dbg.values for now.
1636 if (!IsVariadic &&
1637 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1638 FuncArgumentDbgValueKind::Value, N))
1639 return true;
1640 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1641 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1642 // describe stack slot locations.
1643 //
1644 // Consider "int x = 0; int *px = &x;". There are two kinds of
1645 // interesting debug values here after optimization:
1646 //
1647 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1648 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1649 //
1650 // Both describe the direct values of their associated variables.
1651 Dependencies.push_back(N.getNode());
1652 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1653 continue;
1654 }
1655 LocationOps.emplace_back(
1656 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1657 continue;
1658 }
1659
1660 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1661 // Special rules apply for the first dbg.values of parameter variables in a
1662 // function. Identify them by the fact they reference Argument Values, that
1663 // they're parameters, and they are parameters of the current function. We
1664 // need to let them dangle until they get an SDNode.
1665 bool IsParamOfFunc =
1666 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1667 if (IsParamOfFunc)
1668 return false;
1669
1670 // The value is not used in this block yet (or it would have an SDNode).
1671 // We still want the value to appear for the user if possible -- if it has
1672 // an associated VReg, we can refer to that instead.
1673 auto VMI = FuncInfo.ValueMap.find(V);
1674 if (VMI != FuncInfo.ValueMap.end()) {
1675 Register Reg = VMI->second;
1676 // If this is a PHI node, it may be split up into several MI PHI nodes
1677 // (in FunctionLoweringInfo::set).
1678 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1679 V->getType(), std::nullopt);
1680 if (RFV.occupiesMultipleRegs()) {
1681 // FIXME: We could potentially support variadic dbg_values here.
1682 if (IsVariadic)
1683 return false;
1684 unsigned Offset = 0;
1685 unsigned BitsToDescribe = 0;
1686 if (auto VarSize = Var->getSizeInBits())
1687 BitsToDescribe = *VarSize;
1688 if (auto Fragment = Expr->getFragmentInfo())
1689 BitsToDescribe = Fragment->SizeInBits;
1690 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1691 // Bail out if all bits are described already.
1692 if (Offset >= BitsToDescribe)
1693 break;
1694 // TODO: handle scalable vectors.
1695 unsigned RegisterSize = RegAndSize.second;
1696 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1697 ? BitsToDescribe - Offset
1698 : RegisterSize;
1699 auto FragmentExpr = DIExpression::createFragmentExpression(
1700 Expr, Offset, FragmentSize);
1701 if (!FragmentExpr)
1702 continue;
1703 SDDbgValue *SDV = DAG.getVRegDbgValue(
1704 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);
1705 DAG.AddDbgValue(SDV, false);
1706 Offset += RegisterSize;
1707 }
1708 return true;
1709 }
1710 // We can use simple vreg locations for variadic dbg_values as well.
1711 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1712 continue;
1713 }
1714 // We failed to create a SDDbgOperand for V.
1715 return false;
1716 }
1717
1718 // We have created a SDDbgOperand for each Value in Values.
1719 assert(!LocationOps.empty());
1720 SDDbgValue *SDV =
1721 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1722 /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
1723 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1724 return true;
1725}
1726
1728 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1729 for (auto &Pair : DanglingDebugInfoMap)
1730 for (auto &DDI : Pair.second)
1731 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1733}
1734
1735/// getCopyFromRegs - If there was virtual register allocated for the value V
1736/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1739 SDValue Result;
1740
1741 if (It != FuncInfo.ValueMap.end()) {
1742 Register InReg = It->second;
1743
1744 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1745 DAG.getDataLayout(), InReg, Ty,
1746 std::nullopt); // This is not an ABI copy.
1747 SDValue Chain = DAG.getEntryNode();
1748 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1749 V);
1750 resolveDanglingDebugInfo(V, Result);
1751 }
1752
1753 return Result;
1754}
1755
1756/// getValue - Return an SDValue for the given Value.
1758 // If we already have an SDValue for this value, use it. It's important
1759 // to do this first, so that we don't create a CopyFromReg if we already
1760 // have a regular SDValue.
1761 SDValue &N = NodeMap[V];
1762 if (N.getNode()) return N;
1763
1764 // If there's a virtual register allocated and initialized for this
1765 // value, use it.
1766 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1767 return copyFromReg;
1768
1769 // Otherwise create a new SDValue and remember it.
1770 SDValue Val = getValueImpl(V);
1771 NodeMap[V] = Val;
1773 return Val;
1774}
1775
1776/// getNonRegisterValue - Return an SDValue for the given Value, but
1777/// don't look in FuncInfo.ValueMap for a virtual register.
1779 // If we already have an SDValue for this value, use it.
1780 SDValue &N = NodeMap[V];
1781 if (N.getNode()) {
1782 if (isIntOrFPConstant(N)) {
1783 // Remove the debug location from the node as the node is about to be used
1784 // in a location which may differ from the original debug location. This
1785 // is relevant to Constant and ConstantFP nodes because they can appear
1786 // as constant expressions inside PHI nodes.
1787 N->setDebugLoc(DebugLoc());
1788 }
1789 return N;
1790 }
1791
1792 // Otherwise create a new SDValue and remember it.
1793 SDValue Val = getValueImpl(V);
1794 NodeMap[V] = Val;
1796 return Val;
1797}
1798
1799/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1800/// Create an SDValue for the given value.
1802 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1803
1804 if (const Constant *C = dyn_cast<Constant>(V)) {
1805 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1806
1807 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) {
1808 SDLoc DL = getCurSDLoc();
1809
1810 // DAG.getConstant() may attempt to legalise the vector constant which can
1811 // significantly change the combines applied to the DAG. To reduce the
1812 // divergence when enabling ConstantInt based vectors we try to construct
1813 // the DAG in the same way as shufflevector based splats. TODO: The
1814 // divergence sometimes leads to better optimisations. Ideally we should
1815 // prevent DAG.getConstant() from legalising too early but there are some
1816 // degradations preventing this.
1817 if (VT.isScalableVector())
1818 return DAG.getNode(
1819 ISD::SPLAT_VECTOR, DL, VT,
1820 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));
1821 if (VT.isFixedLengthVector())
1822 return DAG.getSplatBuildVector(
1823 VT, DL,
1824 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));
1825 return DAG.getConstant(*CI, DL, VT);
1826 }
1827
1828 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1829 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1830
1831 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
1832 return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT,
1833 getValue(CPA->getPointer()), getValue(CPA->getKey()),
1834 getValue(CPA->getAddrDiscriminator()),
1835 getValue(CPA->getDiscriminator()));
1836 }
1837
1839 return DAG.getConstant(0, getCurSDLoc(), VT);
1840
1841 if (match(C, m_VScale()))
1842 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1843
1844 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1845 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1846
1847 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1848 return isa<PoisonValue>(C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1849
1850 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1851 visit(CE->getOpcode(), *CE);
1852 SDValue N1 = NodeMap[V];
1853 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1854 return N1;
1855 }
1856
1858 SmallVector<SDValue, 4> Constants;
1859 for (const Use &U : C->operands()) {
1860 SDNode *Val = getValue(U).getNode();
1861 // If the operand is an empty aggregate, there are no values.
1862 if (!Val) continue;
1863 // Add each leaf value from the operand to the Constants list
1864 // to form a flattened list of all the values.
1865 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1866 Constants.push_back(SDValue(Val, i));
1867 }
1868
1869 return DAG.getMergeValues(Constants, getCurSDLoc());
1870 }
1871
1872 if (const ConstantDataSequential *CDS =
1875 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1876 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1877 // Add each leaf value from the operand to the Constants list
1878 // to form a flattened list of all the values.
1879 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1880 Ops.push_back(SDValue(Val, i));
1881 }
1882
1883 if (isa<ArrayType>(CDS->getType()))
1884 return DAG.getMergeValues(Ops, getCurSDLoc());
1885 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1886 }
1887
1888 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1890 "Unknown struct or array constant!");
1891
1892 SmallVector<EVT, 4> ValueVTs;
1893 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1894 unsigned NumElts = ValueVTs.size();
1895 if (NumElts == 0)
1896 return SDValue(); // empty struct
1897 SmallVector<SDValue, 4> Constants(NumElts);
1898 for (unsigned i = 0; i != NumElts; ++i) {
1899 EVT EltVT = ValueVTs[i];
1900 if (isa<UndefValue>(C))
1901 Constants[i] = DAG.getUNDEF(EltVT);
1902 else if (EltVT.isFloatingPoint())
1903 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1904 else
1905 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1906 }
1907
1908 return DAG.getMergeValues(Constants, getCurSDLoc());
1909 }
1910
1911 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1912 return DAG.getBlockAddress(BA, VT);
1913
1914 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1915 return getValue(Equiv->getGlobalValue());
1916
1917 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1918 return getValue(NC->getGlobalValue());
1919
1920 if (VT == MVT::aarch64svcount) {
1921 assert(C->isNullValue() && "Can only zero this target type!");
1922 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1923 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1924 }
1925
1926 if (VT.isRISCVVectorTuple()) {
1927 assert(C->isNullValue() && "Can only zero this target type!");
1928 return DAG.getNode(
1929 ISD::BITCAST, getCurSDLoc(), VT,
1930 DAG.getNode(
1932 EVT::getVectorVT(*DAG.getContext(), MVT::i8,
1933 VT.getSizeInBits().getKnownMinValue() / 8, true),
1934 DAG.getConstant(0, getCurSDLoc(), MVT::getIntegerVT(8))));
1935 }
1936
1937 VectorType *VecTy = cast<VectorType>(V->getType());
1938
1939 // Now that we know the number and type of the elements, get that number of
1940 // elements into the Ops array based on what kind of constant it is.
1941 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1943 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1944 for (unsigned i = 0; i != NumElements; ++i)
1945 Ops.push_back(getValue(CV->getOperand(i)));
1946
1947 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1948 }
1949
1951 EVT EltVT =
1952 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1953
1954 SDValue Op;
1955 if (EltVT.isFloatingPoint())
1956 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1957 else
1958 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1959
1960 return DAG.getSplat(VT, getCurSDLoc(), Op);
1961 }
1962
1963 llvm_unreachable("Unknown vector constant");
1964 }
1965
1966 // If this is a static alloca, generate it as the frameindex instead of
1967 // computation.
1968 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1970 FuncInfo.StaticAllocaMap.find(AI);
1971 if (SI != FuncInfo.StaticAllocaMap.end())
1972 return DAG.getFrameIndex(
1973 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1974 }
1975
1976 // If this is an instruction which fast-isel has deferred, select it now.
1977 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1978 Register InReg = FuncInfo.InitializeRegForValue(Inst);
1979
1980 std::optional<CallingConv::ID> CallConv;
1981 auto *CI = dyn_cast<CallInst>(Inst);
1982 if (CI && !CI->isInlineAsm())
1983 CallConv = CI->getCallingConv();
1984
1985 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1986 Inst->getType(), CallConv);
1987 SDValue Chain = DAG.getEntryNode();
1988 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1989 }
1990
1991 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1992 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1993
1994 if (const auto *BB = dyn_cast<BasicBlock>(V))
1995 return DAG.getBasicBlock(FuncInfo.getMBB(BB));
1996
1997 llvm_unreachable("Can't get register for value!");
1998}
1999
2000void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
2002 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
2003 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
2004 bool IsSEH = isAsynchronousEHPersonality(Pers);
2005 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
2006 if (IsSEH) {
2007 // For SEH, EHCont Guard needs to know that this catchpad is a target.
2008 CatchPadMBB->setIsEHContTarget(true);
2010 } else
2011 CatchPadMBB->setIsEHScopeEntry();
2012 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
2013 if (IsMSVCCXX || IsCoreCLR)
2014 CatchPadMBB->setIsEHFuncletEntry();
2015}
2016
2017void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
2018 // Update machine-CFG edge.
2019 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());
2020 FuncInfo.MBB->addSuccessor(TargetMBB);
2021
2022 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
2023 bool IsSEH = isAsynchronousEHPersonality(Pers);
2024 if (IsSEH) {
2025 // If this is not a fall-through branch or optimizations are switched off,
2026 // emit the branch.
2027 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
2028 TM.getOptLevel() == CodeGenOptLevel::None)
2029 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2030 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
2031 return;
2032 }
2033
2034 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2035 TargetMBB->setIsEHContTarget(true);
2036 DAG.getMachineFunction().setHasEHContTarget(true);
2037
2038 // Figure out the funclet membership for the catchret's successor.
2039 // This will be used by the FuncletLayout pass to determine how to order the
2040 // BB's.
2041 // A 'catchret' returns to the outer scope's color.
2042 Value *ParentPad = I.getCatchSwitchParentPad();
2043 const BasicBlock *SuccessorColor;
2044 if (isa<ConstantTokenNone>(ParentPad))
2045 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2046 else
2047 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
2048 assert(SuccessorColor && "No parent funclet for catchret!");
2049 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(SuccessorColor);
2050 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2051
2052 // Create the terminator node.
2053 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
2054 getControlRoot(), DAG.getBasicBlock(TargetMBB),
2055 DAG.getBasicBlock(SuccessorColorMBB));
2056 DAG.setRoot(Ret);
2057}
2058
2059void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2060 // Don't emit any special code for the cleanuppad instruction. It just marks
2061 // the start of an EH scope/funclet.
2062 FuncInfo.MBB->setIsEHScopeEntry();
2063 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
2064 if (Pers != EHPersonality::Wasm_CXX) {
2065 FuncInfo.MBB->setIsEHFuncletEntry();
2066 FuncInfo.MBB->setIsCleanupFuncletEntry();
2067 }
2068}
2069
2070/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2071/// many places it could ultimately go. In the IR, we have a single unwind
2072/// destination, but in the machine CFG, we enumerate all the possible blocks.
2073/// This function skips over imaginary basic blocks that hold catchswitch
2074/// instructions, and finds all the "real" machine
2075/// basic block destinations. As those destinations may not be successors of
2076/// EHPadBB, here we also calculate the edge probability to those destinations.
2077/// The passed-in Prob is the edge probability to EHPadBB.
2079 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2080 BranchProbability Prob,
2081 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2082 &UnwindDests) {
2083 EHPersonality Personality =
2085 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2086 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2087 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2088 bool IsSEH = isAsynchronousEHPersonality(Personality);
2089
2090 while (EHPadBB) {
2092 BasicBlock *NewEHPadBB = nullptr;
2093 if (isa<LandingPadInst>(Pad)) {
2094 // Stop on landingpads. They are not funclets.
2095 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2096 break;
2097 } else if (isa<CleanupPadInst>(Pad)) {
2098 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2099 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2100 // which always catches an exception.
2101 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2102 UnwindDests.back().first->setIsEHScopeEntry();
2103 // In Wasm, EH scopes are not funclets
2104 if (!IsWasmCXX)
2105 UnwindDests.back().first->setIsEHFuncletEntry();
2106 break;
2107 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2108 // Add the catchpad handlers to the possible destinations.
2109 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2110 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2111 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2112 if (IsMSVCCXX || IsCoreCLR)
2113 UnwindDests.back().first->setIsEHFuncletEntry();
2114 if (!IsSEH)
2115 UnwindDests.back().first->setIsEHScopeEntry();
2116 }
2117 NewEHPadBB = CatchSwitch->getUnwindDest();
2118 } else {
2119 continue;
2120 }
2121
2122 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2123 if (BPI && NewEHPadBB)
2124 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2125 EHPadBB = NewEHPadBB;
2126 }
2127}
2128
2129void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2130 // Update successor info.
2132 auto UnwindDest = I.getUnwindDest();
2133 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2134 BranchProbability UnwindDestProb =
2135 (BPI && UnwindDest)
2136 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2138 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2139 for (auto &UnwindDest : UnwindDests) {
2140 UnwindDest.first->setIsEHPad();
2141 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2142 }
2143 FuncInfo.MBB->normalizeSuccProbs();
2144
2145 // Create the terminator node.
2146 MachineBasicBlock *CleanupPadMBB =
2147 FuncInfo.getMBB(I.getCleanupPad()->getParent());
2148 SDValue Ret = DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other,
2149 getControlRoot(), DAG.getBasicBlock(CleanupPadMBB));
2150 DAG.setRoot(Ret);
2151}
2152
2153void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2154 report_fatal_error("visitCatchSwitch not yet implemented!");
2155}
2156
2157void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2158 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2159 auto &DL = DAG.getDataLayout();
2160 SDValue Chain = getControlRoot();
2163
2164 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2165 // lower
2166 //
2167 // %val = call <ty> @llvm.experimental.deoptimize()
2168 // ret <ty> %val
2169 //
2170 // differently.
2171 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2173 return;
2174 }
2175
2176 if (!FuncInfo.CanLowerReturn) {
2177 Register DemoteReg = FuncInfo.DemoteRegister;
2178
2179 // Emit a store of the return value through the virtual register.
2180 // Leave Outs empty so that LowerReturn won't try to load return
2181 // registers the usual way.
2182 MVT PtrValueVT = TLI.getPointerTy(DL, DL.getAllocaAddrSpace());
2183 SDValue RetPtr =
2184 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVT);
2185 SDValue RetOp = getValue(I.getOperand(0));
2186
2187 SmallVector<EVT, 4> ValueVTs, MemVTs;
2188 SmallVector<uint64_t, 4> Offsets;
2189 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2190 &Offsets, 0);
2191 unsigned NumValues = ValueVTs.size();
2192
2193 SmallVector<SDValue, 4> Chains(NumValues);
2194 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2195 for (unsigned i = 0; i != NumValues; ++i) {
2196 // An aggregate return value cannot wrap around the address space, so
2197 // offsets to its parts don't wrap either.
2198 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
2199 TypeSize::getFixed(Offsets[i]));
2200
2201 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2202 if (MemVTs[i] != ValueVTs[i])
2203 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2204 Chains[i] = DAG.getStore(
2205 Chain, getCurSDLoc(), Val,
2206 // FIXME: better loc info would be nice.
2207 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
2208 commonAlignment(BaseAlign, Offsets[i]));
2209 }
2210
2211 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
2212 MVT::Other, Chains);
2213 } else if (I.getNumOperands() != 0) {
2215 ComputeValueTypes(DL, I.getOperand(0)->getType(), Types);
2216 unsigned NumValues = Types.size();
2217 if (NumValues) {
2218 SDValue RetOp = getValue(I.getOperand(0));
2219
2220 const Function *F = I.getParent()->getParent();
2221
2222 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2223 I.getOperand(0)->getType(), F->getCallingConv(),
2224 /*IsVarArg*/ false, DL);
2225
2226 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2227 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2228 ExtendKind = ISD::SIGN_EXTEND;
2229 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2230 ExtendKind = ISD::ZERO_EXTEND;
2231
2232 LLVMContext &Context = F->getContext();
2233 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2234
2235 for (unsigned j = 0; j != NumValues; ++j) {
2236 EVT VT = TLI.getValueType(DL, Types[j]);
2237
2238 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2239 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2240
2241 CallingConv::ID CC = F->getCallingConv();
2242
2243 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2244 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2245 SmallVector<SDValue, 4> Parts(NumParts);
2247 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2248 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2249
2250 // 'inreg' on function refers to return value
2251 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2252 if (RetInReg)
2253 Flags.setInReg();
2254
2255 if (I.getOperand(0)->getType()->isPointerTy()) {
2256 Flags.setPointer();
2257 Flags.setPointerAddrSpace(
2258 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2259 }
2260
2261 if (NeedsRegBlock) {
2262 Flags.setInConsecutiveRegs();
2263 if (j == NumValues - 1)
2264 Flags.setInConsecutiveRegsLast();
2265 }
2266
2267 // Propagate extension type if any
2268 if (ExtendKind == ISD::SIGN_EXTEND)
2269 Flags.setSExt();
2270 else if (ExtendKind == ISD::ZERO_EXTEND)
2271 Flags.setZExt();
2272 else if (F->getAttributes().hasRetAttr(Attribute::NoExt))
2273 Flags.setNoExt();
2274
2275 for (unsigned i = 0; i < NumParts; ++i) {
2276 Outs.push_back(ISD::OutputArg(Flags,
2277 Parts[i].getValueType().getSimpleVT(),
2278 VT, Types[j], 0, 0));
2279 OutVals.push_back(Parts[i]);
2280 }
2281 }
2282 }
2283 }
2284
2285 // Push in swifterror virtual register as the last element of Outs. This makes
2286 // sure swifterror virtual register will be returned in the swifterror
2287 // physical register.
2288 const Function *F = I.getParent()->getParent();
2289 if (TLI.supportSwiftError() &&
2290 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2291 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2292 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2293 Flags.setSwiftError();
2294 Outs.push_back(ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
2295 /*argvt=*/EVT(TLI.getPointerTy(DL)),
2296 PointerType::getUnqual(*DAG.getContext()),
2297 /*origidx=*/1, /*partOffs=*/0));
2298 // Create SDNode for the swifterror virtual register.
2299 OutVals.push_back(
2300 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2301 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2302 EVT(TLI.getPointerTy(DL))));
2303 }
2304
2305 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2306 CallingConv::ID CallConv =
2307 DAG.getMachineFunction().getFunction().getCallingConv();
2308 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2309 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2310
2311 // Verify that the target's LowerReturn behaved as expected.
2312 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2313 "LowerReturn didn't return a valid chain!");
2314
2315 // Update the DAG with the new chain value resulting from return lowering.
2316 DAG.setRoot(Chain);
2317}
2318
2319/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2320/// created for it, emit nodes to copy the value into the virtual
2321/// registers.
2323 // Skip empty types
2324 if (V->getType()->isEmptyTy())
2325 return;
2326
2328 if (VMI != FuncInfo.ValueMap.end()) {
2329 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2330 "Unused value assigned virtual registers!");
2331 CopyValueToVirtualRegister(V, VMI->second);
2332 }
2333}
2334
2335/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2336/// the current basic block, add it to ValueMap now so that we'll get a
2337/// CopyTo/FromReg.
2339 // No need to export constants.
2340 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2341
2342 // Already exported?
2343 if (FuncInfo.isExportedInst(V)) return;
2344
2345 Register Reg = FuncInfo.InitializeRegForValue(V);
2347}
2348
2350 const BasicBlock *FromBB) {
2351 // The operands of the setcc have to be in this block. We don't know
2352 // how to export them from some other block.
2353 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2354 // Can export from current BB.
2355 if (VI->getParent() == FromBB)
2356 return true;
2357
2358 // Is already exported, noop.
2359 return FuncInfo.isExportedInst(V);
2360 }
2361
2362 // If this is an argument, we can export it if the BB is the entry block or
2363 // if it is already exported.
2364 if (isa<Argument>(V)) {
2365 if (FromBB->isEntryBlock())
2366 return true;
2367
2368 // Otherwise, can only export this if it is already exported.
2369 return FuncInfo.isExportedInst(V);
2370 }
2371
2372 // Otherwise, constants can always be exported.
2373 return true;
2374}
2375
2376/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2378SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2379 const MachineBasicBlock *Dst) const {
2381 const BasicBlock *SrcBB = Src->getBasicBlock();
2382 const BasicBlock *DstBB = Dst->getBasicBlock();
2383 if (!BPI) {
2384 // If BPI is not available, set the default probability as 1 / N, where N is
2385 // the number of successors.
2386 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2387 return BranchProbability(1, SuccSize);
2388 }
2389 return BPI->getEdgeProbability(SrcBB, DstBB);
2390}
2391
2392void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2393 MachineBasicBlock *Dst,
2394 BranchProbability Prob) {
2395 if (!FuncInfo.BPI)
2396 Src->addSuccessorWithoutProb(Dst);
2397 else {
2398 if (Prob.isUnknown())
2399 Prob = getEdgeProbability(Src, Dst);
2400 Src->addSuccessor(Dst, Prob);
2401 }
2402}
2403
2404static bool InBlock(const Value *V, const BasicBlock *BB) {
2405 if (const Instruction *I = dyn_cast<Instruction>(V))
2406 return I->getParent() == BB;
2407 return true;
2408}
2409
2410/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2411/// This function emits a branch and is used at the leaves of an OR or an
2412/// AND operator tree.
2413void
2416 MachineBasicBlock *FBB,
2417 MachineBasicBlock *CurBB,
2418 MachineBasicBlock *SwitchBB,
2419 BranchProbability TProb,
2420 BranchProbability FProb,
2421 bool InvertCond) {
2422 const BasicBlock *BB = CurBB->getBasicBlock();
2423
2424 // If the leaf of the tree is a comparison, merge the condition into
2425 // the caseblock.
2426 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2427 // The operands of the cmp have to be in this block. We don't know
2428 // how to export them from some other block. If this is the first block
2429 // of the sequence, no exporting is needed.
2430 if (CurBB == SwitchBB ||
2431 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2432 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2433 ISD::CondCode Condition;
2434 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2435 ICmpInst::Predicate Pred =
2436 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2437 Condition = getICmpCondCode(Pred);
2438 } else {
2439 const FCmpInst *FC = cast<FCmpInst>(Cond);
2440 FCmpInst::Predicate Pred =
2441 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2442 Condition = getFCmpCondCode(Pred);
2443 if (TM.Options.NoNaNsFPMath)
2444 Condition = getFCmpCodeWithoutNaN(Condition);
2445 }
2446
2447 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2448 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2449 SL->SwitchCases.push_back(CB);
2450 return;
2451 }
2452 }
2453
2454 // Create a CaseBlock record representing this branch.
2455 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2456 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2457 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2458 SL->SwitchCases.push_back(CB);
2459}
2460
2461// Collect dependencies on V recursively. This is used for the cost analysis in
2462// `shouldKeepJumpConditionsTogether`.
2466 unsigned Depth = 0) {
2467 // Return false if we have an incomplete count.
2469 return false;
2470
2471 auto *I = dyn_cast<Instruction>(V);
2472 if (I == nullptr)
2473 return true;
2474
2475 if (Necessary != nullptr) {
2476 // This instruction is necessary for the other side of the condition so
2477 // don't count it.
2478 if (Necessary->contains(I))
2479 return true;
2480 }
2481
2482 // Already added this dep.
2483 if (!Deps->try_emplace(I, false).second)
2484 return true;
2485
2486 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2487 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2488 Depth + 1))
2489 return false;
2490 return true;
2491}
2492
2495 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2497 if (I.getNumSuccessors() != 2)
2498 return false;
2499
2500 if (!I.isConditional())
2501 return false;
2502
2503 if (Params.BaseCost < 0)
2504 return false;
2505
2506 // Baseline cost.
2507 InstructionCost CostThresh = Params.BaseCost;
2508
2509 BranchProbabilityInfo *BPI = nullptr;
2510 if (Params.LikelyBias || Params.UnlikelyBias)
2511 BPI = FuncInfo.BPI;
2512 if (BPI != nullptr) {
2513 // See if we are either likely to get an early out or compute both lhs/rhs
2514 // of the condition.
2515 BasicBlock *IfFalse = I.getSuccessor(0);
2516 BasicBlock *IfTrue = I.getSuccessor(1);
2517
2518 std::optional<bool> Likely;
2519 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2520 Likely = true;
2521 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2522 Likely = false;
2523
2524 if (Likely) {
2525 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2526 // Its likely we will have to compute both lhs and rhs of condition
2527 CostThresh += Params.LikelyBias;
2528 else {
2529 if (Params.UnlikelyBias < 0)
2530 return false;
2531 // Its likely we will get an early out.
2532 CostThresh -= Params.UnlikelyBias;
2533 }
2534 }
2535 }
2536
2537 if (CostThresh <= 0)
2538 return false;
2539
2540 // Collect "all" instructions that lhs condition is dependent on.
2541 // Use map for stable iteration (to avoid non-determanism of iteration of
2542 // SmallPtrSet). The `bool` value is just a dummy.
2544 collectInstructionDeps(&LhsDeps, Lhs);
2545 // Collect "all" instructions that rhs condition is dependent on AND are
2546 // dependencies of lhs. This gives us an estimate on which instructions we
2547 // stand to save by splitting the condition.
2548 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2549 return false;
2550 // Add the compare instruction itself unless its a dependency on the LHS.
2551 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2552 if (!LhsDeps.contains(RhsI))
2553 RhsDeps.try_emplace(RhsI, false);
2554
2555 const auto &TLI = DAG.getTargetLoweringInfo();
2556 const auto &TTI =
2557 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
2558
2559 InstructionCost CostOfIncluding = 0;
2560 // See if this instruction will need to computed independently of whether RHS
2561 // is.
2562 Value *BrCond = I.getCondition();
2563 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2564 for (const auto *U : Ins->users()) {
2565 // If user is independent of RHS calculation we don't need to count it.
2566 if (auto *UIns = dyn_cast<Instruction>(U))
2567 if (UIns != BrCond && !RhsDeps.contains(UIns))
2568 return false;
2569 }
2570 return true;
2571 };
2572
2573 // Prune instructions from RHS Deps that are dependencies of unrelated
2574 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2575 // arbitrary and just meant to cap the how much time we spend in the pruning
2576 // loop. Its highly unlikely to come into affect.
2577 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2578 // Stop after a certain point. No incorrectness from including too many
2579 // instructions.
2580 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2581 const Instruction *ToDrop = nullptr;
2582 for (const auto &InsPair : RhsDeps) {
2583 if (!ShouldCountInsn(InsPair.first)) {
2584 ToDrop = InsPair.first;
2585 break;
2586 }
2587 }
2588 if (ToDrop == nullptr)
2589 break;
2590 RhsDeps.erase(ToDrop);
2591 }
2592
2593 for (const auto &InsPair : RhsDeps) {
2594 // Finally accumulate latency that we can only attribute to computing the
2595 // RHS condition. Use latency because we are essentially trying to calculate
2596 // the cost of the dependency chain.
2597 // Possible TODO: We could try to estimate ILP and make this more precise.
2598 CostOfIncluding +=
2599 TTI.getInstructionCost(InsPair.first, TargetTransformInfo::TCK_Latency);
2600
2601 if (CostOfIncluding > CostThresh)
2602 return false;
2603 }
2604 return true;
2605}
2606
2609 MachineBasicBlock *FBB,
2610 MachineBasicBlock *CurBB,
2611 MachineBasicBlock *SwitchBB,
2613 BranchProbability TProb,
2614 BranchProbability FProb,
2615 bool InvertCond) {
2616 // Skip over not part of the tree and remember to invert op and operands at
2617 // next level.
2618 Value *NotCond;
2619 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2620 InBlock(NotCond, CurBB->getBasicBlock())) {
2621 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2622 !InvertCond);
2623 return;
2624 }
2625
2627 const Value *BOpOp0, *BOpOp1;
2628 // Compute the effective opcode for Cond, taking into account whether it needs
2629 // to be inverted, e.g.
2630 // and (not (or A, B)), C
2631 // gets lowered as
2632 // and (and (not A, not B), C)
2634 if (BOp) {
2635 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2636 ? Instruction::And
2637 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2638 ? Instruction::Or
2640 if (InvertCond) {
2641 if (BOpc == Instruction::And)
2642 BOpc = Instruction::Or;
2643 else if (BOpc == Instruction::Or)
2644 BOpc = Instruction::And;
2645 }
2646 }
2647
2648 // If this node is not part of the or/and tree, emit it as a branch.
2649 // Note that all nodes in the tree should have same opcode.
2650 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2651 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2652 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2653 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2654 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2655 TProb, FProb, InvertCond);
2656 return;
2657 }
2658
2659 // Create TmpBB after CurBB.
2660 MachineFunction::iterator BBI(CurBB);
2661 MachineFunction &MF = DAG.getMachineFunction();
2663 CurBB->getParent()->insert(++BBI, TmpBB);
2664
2665 if (Opc == Instruction::Or) {
2666 // Codegen X | Y as:
2667 // BB1:
2668 // jmp_if_X TBB
2669 // jmp TmpBB
2670 // TmpBB:
2671 // jmp_if_Y TBB
2672 // jmp FBB
2673 //
2674
2675 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2676 // The requirement is that
2677 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2678 // = TrueProb for original BB.
2679 // Assuming the original probabilities are A and B, one choice is to set
2680 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2681 // A/(1+B) and 2B/(1+B). This choice assumes that
2682 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2683 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2684 // TmpBB, but the math is more complicated.
2685
2686 auto NewTrueProb = TProb / 2;
2687 auto NewFalseProb = TProb / 2 + FProb;
2688 // Emit the LHS condition.
2689 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2690 NewFalseProb, InvertCond);
2691
2692 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2693 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2695 // Emit the RHS condition into TmpBB.
2696 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2697 Probs[1], InvertCond);
2698 } else {
2699 assert(Opc == Instruction::And && "Unknown merge op!");
2700 // Codegen X & Y as:
2701 // BB1:
2702 // jmp_if_X TmpBB
2703 // jmp FBB
2704 // TmpBB:
2705 // jmp_if_Y TBB
2706 // jmp FBB
2707 //
2708 // This requires creation of TmpBB after CurBB.
2709
2710 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2711 // The requirement is that
2712 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2713 // = FalseProb for original BB.
2714 // Assuming the original probabilities are A and B, one choice is to set
2715 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2716 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2717 // TrueProb for BB1 * FalseProb for TmpBB.
2718
2719 auto NewTrueProb = TProb + FProb / 2;
2720 auto NewFalseProb = FProb / 2;
2721 // Emit the LHS condition.
2722 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2723 NewFalseProb, InvertCond);
2724
2725 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2726 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2728 // Emit the RHS condition into TmpBB.
2729 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2730 Probs[1], InvertCond);
2731 }
2732}
2733
2734/// If the set of cases should be emitted as a series of branches, return true.
2735/// If we should emit this as a bunch of and/or'd together conditions, return
2736/// false.
2737bool
2738SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2739 if (Cases.size() != 2) return true;
2740
2741 // If this is two comparisons of the same values or'd or and'd together, they
2742 // will get folded into a single comparison, so don't emit two blocks.
2743 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2744 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2745 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2746 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2747 return false;
2748 }
2749
2750 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2751 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2752 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2753 Cases[0].CC == Cases[1].CC &&
2754 isa<Constant>(Cases[0].CmpRHS) &&
2755 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2756 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2757 return false;
2758 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2759 return false;
2760 }
2761
2762 return true;
2763}
2764
2765void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2767
2768 // Update machine-CFG edges.
2769 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2770
2771 if (I.isUnconditional()) {
2772 // Update machine-CFG edges.
2773 BrMBB->addSuccessor(Succ0MBB);
2774
2775 // If this is not a fall-through branch or optimizations are switched off,
2776 // emit the branch.
2777 if (Succ0MBB != NextBlock(BrMBB) ||
2779 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2780 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2781 setValue(&I, Br);
2782 DAG.setRoot(Br);
2783 }
2784
2785 return;
2786 }
2787
2788 // If this condition is one of the special cases we handle, do special stuff
2789 // now.
2790 const Value *CondVal = I.getCondition();
2791 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(I.getSuccessor(1));
2792
2793 // If this is a series of conditions that are or'd or and'd together, emit
2794 // this as a sequence of branches instead of setcc's with and/or operations.
2795 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2796 // unpredictable branches, and vector extracts because those jumps are likely
2797 // expensive for any target), this should improve performance.
2798 // For example, instead of something like:
2799 // cmp A, B
2800 // C = seteq
2801 // cmp D, E
2802 // F = setle
2803 // or C, F
2804 // jnz foo
2805 // Emit:
2806 // cmp A, B
2807 // je foo
2808 // cmp D, E
2809 // jle foo
2810 bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
2811 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2812 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2813 BOp->hasOneUse() && !IsUnpredictable) {
2814 Value *Vec;
2815 const Value *BOp0, *BOp1;
2817 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2818 Opcode = Instruction::And;
2819 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2820 Opcode = Instruction::Or;
2821
2822 if (Opcode &&
2823 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2824 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2826 FuncInfo, I, Opcode, BOp0, BOp1,
2827 DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2828 Opcode, BOp0, BOp1))) {
2829 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2830 getEdgeProbability(BrMBB, Succ0MBB),
2831 getEdgeProbability(BrMBB, Succ1MBB),
2832 /*InvertCond=*/false);
2833 // If the compares in later blocks need to use values not currently
2834 // exported from this block, export them now. This block should always
2835 // be the first entry.
2836 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2837
2838 // Allow some cases to be rejected.
2839 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2840 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2841 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2842 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2843 }
2844
2845 // Emit the branch for this block.
2846 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2847 SL->SwitchCases.erase(SL->SwitchCases.begin());
2848 return;
2849 }
2850
2851 // Okay, we decided not to do this, remove any inserted MBB's and clear
2852 // SwitchCases.
2853 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2854 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2855
2856 SL->SwitchCases.clear();
2857 }
2858 }
2859
2860 // Create a CaseBlock record representing this branch.
2861 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2862 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2864 IsUnpredictable);
2865
2866 // Use visitSwitchCase to actually insert the fast branch sequence for this
2867 // cond branch.
2868 visitSwitchCase(CB, BrMBB);
2869}
2870
2871/// visitSwitchCase - Emits the necessary code to represent a single node in
2872/// the binary search tree resulting from lowering a switch instruction.
2874 MachineBasicBlock *SwitchBB) {
2875 SDValue Cond;
2876 SDValue CondLHS = getValue(CB.CmpLHS);
2877 SDLoc dl = CB.DL;
2878
2879 if (CB.CC == ISD::SETTRUE) {
2880 // Branch or fall through to TrueBB.
2881 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2882 SwitchBB->normalizeSuccProbs();
2883 if (CB.TrueBB != NextBlock(SwitchBB)) {
2884 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2885 DAG.getBasicBlock(CB.TrueBB)));
2886 }
2887 return;
2888 }
2889
2890 auto &TLI = DAG.getTargetLoweringInfo();
2891 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2892
2893 // Build the setcc now.
2894 if (!CB.CmpMHS) {
2895 // Fold "(X == true)" to X and "(X == false)" to !X to
2896 // handle common cases produced by branch lowering.
2897 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2898 CB.CC == ISD::SETEQ)
2899 Cond = CondLHS;
2900 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2901 CB.CC == ISD::SETEQ) {
2902 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2903 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2904 } else {
2905 SDValue CondRHS = getValue(CB.CmpRHS);
2906
2907 // If a pointer's DAG type is larger than its memory type then the DAG
2908 // values are zero-extended. This breaks signed comparisons so truncate
2909 // back to the underlying type before doing the compare.
2910 if (CondLHS.getValueType() != MemVT) {
2911 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2912 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2913 }
2914 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2915 }
2916 } else {
2917 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2918
2919 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2920 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2921
2922 SDValue CmpOp = getValue(CB.CmpMHS);
2923 EVT VT = CmpOp.getValueType();
2924
2925 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2926 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2927 ISD::SETLE);
2928 } else {
2929 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2930 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2931 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2932 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2933 }
2934 }
2935
2936 // Update successor info
2937 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2938 // TrueBB and FalseBB are always different unless the incoming IR is
2939 // degenerate. This only happens when running llc on weird IR.
2940 if (CB.TrueBB != CB.FalseBB)
2941 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2942 SwitchBB->normalizeSuccProbs();
2943
2944 // If the lhs block is the next block, invert the condition so that we can
2945 // fall through to the lhs instead of the rhs block.
2946 if (CB.TrueBB == NextBlock(SwitchBB)) {
2947 std::swap(CB.TrueBB, CB.FalseBB);
2948 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2949 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2950 }
2951
2952 SDNodeFlags Flags;
2954 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
2955 Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
2956
2957 setValue(CurInst, BrCond);
2958
2959 // Insert the false branch. Do this even if it's a fall through branch,
2960 // this makes it easier to do DAG optimizations which require inverting
2961 // the branch condition.
2962 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2963 DAG.getBasicBlock(CB.FalseBB));
2964
2965 DAG.setRoot(BrCond);
2966}
2967
2968/// visitJumpTable - Emit JumpTable node in the current MBB
2970 // Emit the code for the jump table
2971 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2972 assert(JT.Reg && "Should lower JT Header first!");
2973 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DAG.getDataLayout());
2974 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
2975 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2976 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2977 Index.getValue(1), Table, Index);
2978 DAG.setRoot(BrJumpTable);
2979}
2980
2981/// visitJumpTableHeader - This function emits necessary code to produce index
2982/// in the JumpTable from switch case.
2984 JumpTableHeader &JTH,
2985 MachineBasicBlock *SwitchBB) {
2986 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2987 const SDLoc &dl = *JT.SL;
2988
2989 // Subtract the lowest switch case value from the value being switched on.
2990 SDValue SwitchOp = getValue(JTH.SValue);
2991 EVT VT = SwitchOp.getValueType();
2992 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2993 DAG.getConstant(JTH.First, dl, VT));
2994
2995 // The SDNode we just created, which holds the value being switched on minus
2996 // the smallest case value, needs to be copied to a virtual register so it
2997 // can be used as an index into the jump table in a subsequent basic block.
2998 // This value may be smaller or larger than the target's pointer type, and
2999 // therefore require extension or truncating.
3000 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3001 SwitchOp =
3002 DAG.getZExtOrTrunc(Sub, dl, TLI.getJumpTableRegTy(DAG.getDataLayout()));
3003
3004 Register JumpTableReg =
3005 FuncInfo.CreateReg(TLI.getJumpTableRegTy(DAG.getDataLayout()));
3006 SDValue CopyTo =
3007 DAG.getCopyToReg(getControlRoot(), dl, JumpTableReg, SwitchOp);
3008 JT.Reg = JumpTableReg;
3009
3010 if (!JTH.FallthroughUnreachable) {
3011 // Emit the range check for the jump table, and branch to the default block
3012 // for the switch statement if the value being switched on exceeds the
3013 // largest case in the switch.
3014 SDValue CMP = DAG.getSetCC(
3015 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3016 Sub.getValueType()),
3017 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3018
3019 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3020 MVT::Other, CopyTo, CMP,
3021 DAG.getBasicBlock(JT.Default));
3022
3023 // Avoid emitting unnecessary branches to the next block.
3024 if (JT.MBB != NextBlock(SwitchBB))
3025 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3026 DAG.getBasicBlock(JT.MBB));
3027
3028 DAG.setRoot(BrCond);
3029 } else {
3030 // Avoid emitting unnecessary branches to the next block.
3031 if (JT.MBB != NextBlock(SwitchBB))
3032 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3033 DAG.getBasicBlock(JT.MBB)));
3034 else
3035 DAG.setRoot(CopyTo);
3036 }
3037}
3038
3039/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3040/// variable if there exists one.
3042 SDValue &Chain) {
3043 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3044 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3045 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3049 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3050 if (Global) {
3051 MachinePointerInfo MPInfo(Global);
3055 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
3056 DAG.setNodeMemRefs(Node, {MemRef});
3057 }
3058 if (PtrTy != PtrMemTy)
3059 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3060 return SDValue(Node, 0);
3061}
3062
3063/// Codegen a new tail for a stack protector check ParentMBB which has had its
3064/// tail spliced into a stack protector check success bb.
3065///
3066/// For a high level explanation of how this fits into the stack protector
3067/// generation see the comment on the declaration of class
3068/// StackProtectorDescriptor.
3070 MachineBasicBlock *ParentBB) {
3071
3072 // First create the loads to the guard/stack slot for the comparison.
3073 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3074 auto &DL = DAG.getDataLayout();
3075 EVT PtrTy = TLI.getFrameIndexTy(DL);
3076 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
3077
3078 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3079 int FI = MFI.getStackProtectorIndex();
3080
3081 SDValue Guard;
3082 SDLoc dl = getCurSDLoc();
3083 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3084 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3085 Align Align = DL.getPrefTypeAlign(
3086 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
3087
3088 // Generate code to load the content of the guard slot.
3089 SDValue GuardVal = DAG.getLoad(
3090 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3091 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
3093
3094 if (TLI.useStackGuardXorFP())
3095 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3096
3097 // If we're using function-based instrumentation, call the guard check
3098 // function
3100 // Get the guard check function from the target and verify it exists since
3101 // we're using function-based instrumentation
3102 const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M);
3103 assert(GuardCheckFn && "Guard check function is null");
3104
3105 // The target provides a guard check function to validate the guard value.
3106 // Generate a call to that function with the content of the guard slot as
3107 // argument.
3108 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3109 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3110
3112 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
3113 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3114 Entry.IsInReg = true;
3115 Args.push_back(Entry);
3116
3119 .setChain(DAG.getEntryNode())
3120 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3121 getValue(GuardCheckFn), std::move(Args));
3122
3123 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3124 DAG.setRoot(Result.second);
3125 return;
3126 }
3127
3128 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3129 // Otherwise, emit a volatile load to retrieve the stack guard value.
3130 SDValue Chain = DAG.getEntryNode();
3131 if (TLI.useLoadStackGuardNode(M)) {
3132 Guard = getLoadStackGuard(DAG, dl, Chain);
3133 } else {
3134 const Value *IRGuard = TLI.getSDagStackGuard(M);
3135 SDValue GuardPtr = getValue(IRGuard);
3136
3137 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3138 MachinePointerInfo(IRGuard, 0), Align,
3140 }
3141
3142 // Perform the comparison via a getsetcc.
3143 SDValue Cmp = DAG.getSetCC(
3144 dl, TLI.getSetCCResultType(DL, *DAG.getContext(), Guard.getValueType()),
3145 Guard, GuardVal, ISD::SETNE);
3146
3147 // If the guard/stackslot do not equal, branch to failure MBB.
3148 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3149 MVT::Other, GuardVal.getOperand(0),
3150 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3151 // Otherwise branch to success MBB.
3152 SDValue Br = DAG.getNode(ISD::BR, dl,
3153 MVT::Other, BrCond,
3154 DAG.getBasicBlock(SPD.getSuccessMBB()));
3155
3156 DAG.setRoot(Br);
3157}
3158
3159/// Codegen the failure basic block for a stack protector check.
3160///
3161/// A failure stack protector machine basic block consists simply of a call to
3162/// __stack_chk_fail().
3163///
3164/// For a high level explanation of how this fits into the stack protector
3165/// generation see the comment on the declaration of class
3166/// StackProtectorDescriptor.
3169
3170 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3171 MachineBasicBlock *ParentBB = SPD.getParentMBB();
3172 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3173 SDValue Chain;
3174
3175 // For -Oz builds with a guard check function, we use function-based
3176 // instrumentation. Otherwise, if we have a guard check function, we call it
3177 // in the failure block.
3178 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M);
3179 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3180 // First create the loads to the guard/stack slot for the comparison.
3181 auto &DL = DAG.getDataLayout();
3182 EVT PtrTy = TLI.getFrameIndexTy(DL);
3183 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
3184
3185 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3186 int FI = MFI.getStackProtectorIndex();
3187
3188 SDLoc dl = getCurSDLoc();
3189 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3190 Align Align = DL.getPrefTypeAlign(
3191 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
3192
3193 // Generate code to load the content of the guard slot.
3194 SDValue GuardVal = DAG.getLoad(
3195 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3196 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
3198
3199 if (TLI.useStackGuardXorFP())
3200 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3201
3202 // The target provides a guard check function to validate the guard value.
3203 // Generate a call to that function with the content of the guard slot as
3204 // argument.
3205 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3206 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3207
3209 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
3210 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3211 Entry.IsInReg = true;
3212 Args.push_back(Entry);
3213
3216 .setChain(DAG.getEntryNode())
3217 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3218 getValue(GuardCheckFn), std::move(Args));
3219
3220 Chain = TLI.LowerCallTo(CLI).second;
3221 } else {
3223 CallOptions.setDiscardResult(true);
3224 Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3225 {}, CallOptions, getCurSDLoc())
3226 .second;
3227 }
3228
3229 // Emit a trap instruction if we are required to do so.
3230 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3231 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3232 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3233
3234 DAG.setRoot(Chain);
3235}
3236
3237/// visitBitTestHeader - This function emits necessary code to produce value
3238/// suitable for "bit tests"
3240 MachineBasicBlock *SwitchBB) {
3241 SDLoc dl = getCurSDLoc();
3242
3243 // Subtract the minimum value.
3244 SDValue SwitchOp = getValue(B.SValue);
3245 EVT VT = SwitchOp.getValueType();
3246 SDValue RangeSub =
3247 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3248
3249 // Determine the type of the test operands.
3250 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3251 bool UsePtrType = false;
3252 if (!TLI.isTypeLegal(VT)) {
3253 UsePtrType = true;
3254 } else {
3255 for (const BitTestCase &Case : B.Cases)
3256 if (!isUIntN(VT.getSizeInBits(), Case.Mask)) {
3257 // Switch table case range are encoded into series of masks.
3258 // Just use pointer type, it's guaranteed to fit.
3259 UsePtrType = true;
3260 break;
3261 }
3262 }
3263 SDValue Sub = RangeSub;
3264 if (UsePtrType) {
3265 VT = TLI.getPointerTy(DAG.getDataLayout());
3266 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3267 }
3268
3269 B.RegVT = VT.getSimpleVT();
3270 B.Reg = FuncInfo.CreateReg(B.RegVT);
3271 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3272
3273 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3274
3275 if (!B.FallthroughUnreachable)
3276 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3277 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3278 SwitchBB->normalizeSuccProbs();
3279
3280 SDValue Root = CopyTo;
3281 if (!B.FallthroughUnreachable) {
3282 // Conditional branch to the default block.
3283 SDValue RangeCmp = DAG.getSetCC(dl,
3284 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3285 RangeSub.getValueType()),
3286 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3287 ISD::SETUGT);
3288
3289 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3290 DAG.getBasicBlock(B.Default));
3291 }
3292
3293 // Avoid emitting unnecessary branches to the next block.
3294 if (MBB != NextBlock(SwitchBB))
3295 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3296
3297 DAG.setRoot(Root);
3298}
3299
3300/// visitBitTestCase - this function produces one "bit test"
3302 MachineBasicBlock *NextMBB,
3303 BranchProbability BranchProbToNext,
3304 Register Reg, BitTestCase &B,
3305 MachineBasicBlock *SwitchBB) {
3306 SDLoc dl = getCurSDLoc();
3307 MVT VT = BB.RegVT;
3308 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3309 SDValue Cmp;
3310 unsigned PopCount = llvm::popcount(B.Mask);
3311 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3312 if (PopCount == 1) {
3313 // Testing for a single bit; just compare the shift count with what it
3314 // would need to be to shift a 1 bit in that position.
3315 Cmp = DAG.getSetCC(
3316 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3317 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3318 ISD::SETEQ);
3319 } else if (PopCount == BB.Range) {
3320 // There is only one zero bit in the range, test for it directly.
3321 Cmp = DAG.getSetCC(
3322 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3323 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3324 } else {
3325 // Make desired shift
3326 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3327 DAG.getConstant(1, dl, VT), ShiftOp);
3328
3329 // Emit bit tests and jumps
3330 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3331 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3332 Cmp = DAG.getSetCC(
3333 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3334 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3335 }
3336
3337 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3338 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3339 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3340 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3341 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3342 // one as they are relative probabilities (and thus work more like weights),
3343 // and hence we need to normalize them to let the sum of them become one.
3344 SwitchBB->normalizeSuccProbs();
3345
3346 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3347 MVT::Other, getControlRoot(),
3348 Cmp, DAG.getBasicBlock(B.TargetBB));
3349
3350 // Avoid emitting unnecessary branches to the next block.
3351 if (NextMBB != NextBlock(SwitchBB))
3352 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3353 DAG.getBasicBlock(NextMBB));
3354
3355 DAG.setRoot(BrAnd);
3356}
3357
3358void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3359 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3360
3361 // Retrieve successors. Look through artificial IR level blocks like
3362 // catchswitch for successors.
3363 MachineBasicBlock *Return = FuncInfo.getMBB(I.getSuccessor(0));
3364 const BasicBlock *EHPadBB = I.getSuccessor(1);
3365 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(EHPadBB);
3366
3367 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3368 // have to do anything here to lower funclet bundles.
3369 failForInvalidBundles(I, "invokes",
3375
3376 const Value *Callee(I.getCalledOperand());
3377 const Function *Fn = dyn_cast<Function>(Callee);
3378 if (isa<InlineAsm>(Callee))
3379 visitInlineAsm(I, EHPadBB);
3380 else if (Fn && Fn->isIntrinsic()) {
3381 switch (Fn->getIntrinsicID()) {
3382 default:
3383 llvm_unreachable("Cannot invoke this intrinsic");
3384 case Intrinsic::donothing:
3385 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3386 case Intrinsic::seh_try_begin:
3387 case Intrinsic::seh_scope_begin:
3388 case Intrinsic::seh_try_end:
3389 case Intrinsic::seh_scope_end:
3390 if (EHPadMBB)
3391 // a block referenced by EH table
3392 // so dtor-funclet not removed by opts
3393 EHPadMBB->setMachineBlockAddressTaken();
3394 break;
3395 case Intrinsic::experimental_patchpoint_void:
3396 case Intrinsic::experimental_patchpoint:
3397 visitPatchpoint(I, EHPadBB);
3398 break;
3399 case Intrinsic::experimental_gc_statepoint:
3401 break;
3402 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3403 // but these intrinsics are special because they can be invoked, so we
3404 // manually lower it to a DAG node here.
3405 case Intrinsic::wasm_throw: {
3407 std::array<SDValue, 4> Ops = {
3408 getControlRoot(), // inchain for the terminator node
3409 DAG.getTargetConstant(Intrinsic::wasm_throw, getCurSDLoc(),
3411 getValue(I.getArgOperand(0)), // tag
3412 getValue(I.getArgOperand(1)) // thrown value
3413 };
3414 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3415 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3416 break;
3417 }
3418 case Intrinsic::wasm_rethrow: {
3419 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3420 std::array<SDValue, 2> Ops = {
3421 getControlRoot(), // inchain for the terminator node
3422 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3423 TLI.getPointerTy(DAG.getDataLayout()))};
3424 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3425 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3426 break;
3427 }
3428 }
3429 } else if (I.hasDeoptState()) {
3430 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3431 // Eventually we will support lowering the @llvm.experimental.deoptimize
3432 // intrinsic, and right now there are no plans to support other intrinsics
3433 // with deopt state.
3434 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3435 } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
3437 } else {
3438 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3439 }
3440
3441 // If the value of the invoke is used outside of its defining block, make it
3442 // available as a virtual register.
3443 // We already took care of the exported value for the statepoint instruction
3444 // during call to the LowerStatepoint.
3445 if (!isa<GCStatepointInst>(I)) {
3447 }
3448
3450 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3451 BranchProbability EHPadBBProb =
3452 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3454 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3455
3456 // Update successor info.
3457 addSuccessorWithProb(InvokeMBB, Return);
3458 for (auto &UnwindDest : UnwindDests) {
3459 UnwindDest.first->setIsEHPad();
3460 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3461 }
3462 InvokeMBB->normalizeSuccProbs();
3463
3464 // Drop into normal successor.
3465 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
3466 DAG.getBasicBlock(Return)));
3467}
3468
3469void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3470 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3471
3472 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3473 // have to do anything here to lower funclet bundles.
3474 failForInvalidBundles(I, "callbrs",
3476
3477 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3478 visitInlineAsm(I);
3480
3481 // Retrieve successors.
3482 SmallPtrSet<BasicBlock *, 8> Dests;
3483 Dests.insert(I.getDefaultDest());
3484 MachineBasicBlock *Return = FuncInfo.getMBB(I.getDefaultDest());
3485
3486 // Update successor info.
3487 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3488 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3489 BasicBlock *Dest = I.getIndirectDest(i);
3490 MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
3491 Target->setIsInlineAsmBrIndirectTarget();
3492 // If we introduce a type of asm goto statement that is permitted to use an
3493 // indirect call instruction to jump to its labels, then we should add a
3494 // call to Target->setMachineBlockAddressTaken() here, to mark the target
3495 // block as requiring a BTI.
3496
3497 Target->setLabelMustBeEmitted();
3498 // Don't add duplicate machine successors.
3499 if (Dests.insert(Dest).second)
3500 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3501 }
3502 CallBrMBB->normalizeSuccProbs();
3503
3504 // Drop into default successor.
3505 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3506 MVT::Other, getControlRoot(),
3507 DAG.getBasicBlock(Return)));
3508}
3509
3510void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3511 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3512}
3513
3514void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3515 assert(FuncInfo.MBB->isEHPad() &&
3516 "Call to landingpad not in landing pad!");
3517
3518 // If there aren't registers to copy the values into (e.g., during SjLj
3519 // exceptions), then don't bother to create these DAG nodes.
3520 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3521 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3522 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3523 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3524 return;
3525
3526 // If landingpad's return type is token type, we don't create DAG nodes
3527 // for its exception pointer and selector value. The extraction of exception
3528 // pointer or selector value from token type landingpads is not currently
3529 // supported.
3530 if (LP.getType()->isTokenTy())
3531 return;
3532
3533 SmallVector<EVT, 2> ValueVTs;
3534 SDLoc dl = getCurSDLoc();
3535 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3536 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3537
3538 // Get the two live-in registers as SDValues. The physregs have already been
3539 // copied into virtual registers.
3540 SDValue Ops[2];
3541 if (FuncInfo.ExceptionPointerVirtReg) {
3542 Ops[0] = DAG.getZExtOrTrunc(
3543 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3544 FuncInfo.ExceptionPointerVirtReg,
3545 TLI.getPointerTy(DAG.getDataLayout())),
3546 dl, ValueVTs[0]);
3547 } else {
3548 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3549 }
3550 Ops[1] = DAG.getZExtOrTrunc(
3551 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3552 FuncInfo.ExceptionSelectorVirtReg,
3553 TLI.getPointerTy(DAG.getDataLayout())),
3554 dl, ValueVTs[1]);
3555
3556 // Merge into one.
3557 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3558 DAG.getVTList(ValueVTs), Ops);
3559 setValue(&LP, Res);
3560}
3561
3564 // Update JTCases.
3565 for (JumpTableBlock &JTB : SL->JTCases)
3566 if (JTB.first.HeaderBB == First)
3567 JTB.first.HeaderBB = Last;
3568
3569 // Update BitTestCases.
3570 for (BitTestBlock &BTB : SL->BitTestCases)
3571 if (BTB.Parent == First)
3572 BTB.Parent = Last;
3573}
3574
3575void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3576 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3577
3578 // Update machine-CFG edges with unique successors.
3580 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3581 BasicBlock *BB = I.getSuccessor(i);
3582 bool Inserted = Done.insert(BB).second;
3583 if (!Inserted)
3584 continue;
3585
3586 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3587 addSuccessorWithProb(IndirectBrMBB, Succ);
3588 }
3589 IndirectBrMBB->normalizeSuccProbs();
3590
3591 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
3592 MVT::Other, getControlRoot(),
3593 getValue(I.getAddress())));
3594}
3595
3596void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3597 if (!I.shouldLowerToTrap(DAG.getTarget().Options.TrapUnreachable,
3598 DAG.getTarget().Options.NoTrapAfterNoreturn))
3599 return;
3600
3601 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3602}
3603
3604void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3605 SDNodeFlags Flags;
3606 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3607 Flags.copyFMF(*FPOp);
3608
3609 SDValue Op = getValue(I.getOperand(0));
3610 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3611 Op, Flags);
3612 setValue(&I, UnNodeValue);
3613}
3614
3615void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3616 SDNodeFlags Flags;
3617 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3618 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3619 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3620 }
3621 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3622 Flags.setExact(ExactOp->isExact());
3623 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3624 Flags.setDisjoint(DisjointOp->isDisjoint());
3625 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3626 Flags.copyFMF(*FPOp);
3627
3628 SDValue Op1 = getValue(I.getOperand(0));
3629 SDValue Op2 = getValue(I.getOperand(1));
3630 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3631 Op1, Op2, Flags);
3632 setValue(&I, BinNodeValue);
3633}
3634
3635void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3636 SDValue Op1 = getValue(I.getOperand(0));
3637 SDValue Op2 = getValue(I.getOperand(1));
3638
3639 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3640 Op1.getValueType(), DAG.getDataLayout());
3641
3642 // Coerce the shift amount to the right type if we can. This exposes the
3643 // truncate or zext to optimization early.
3644 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3646 "Unexpected shift type");
3647 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3648 }
3649
3650 bool nuw = false;
3651 bool nsw = false;
3652 bool exact = false;
3653
3654 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3655
3656 if (const OverflowingBinaryOperator *OFBinOp =
3658 nuw = OFBinOp->hasNoUnsignedWrap();
3659 nsw = OFBinOp->hasNoSignedWrap();
3660 }
3661 if (const PossiblyExactOperator *ExactOp =
3663 exact = ExactOp->isExact();
3664 }
3665 SDNodeFlags Flags;
3666 Flags.setExact(exact);
3667 Flags.setNoSignedWrap(nsw);
3668 Flags.setNoUnsignedWrap(nuw);
3669 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3670 Flags);
3671 setValue(&I, Res);
3672}
3673
3674void SelectionDAGBuilder::visitSDiv(const User &I) {
3675 SDValue Op1 = getValue(I.getOperand(0));
3676 SDValue Op2 = getValue(I.getOperand(1));
3677
3678 SDNodeFlags Flags;
3679 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3680 cast<PossiblyExactOperator>(&I)->isExact());
3681 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3682 Op2, Flags));
3683}
3684
3685void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3686 ICmpInst::Predicate predicate = I.getPredicate();
3687 SDValue Op1 = getValue(I.getOperand(0));
3688 SDValue Op2 = getValue(I.getOperand(1));
3689 ISD::CondCode Opcode = getICmpCondCode(predicate);
3690
3691 auto &TLI = DAG.getTargetLoweringInfo();
3692 EVT MemVT =
3693 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3694
3695 // If a pointer's DAG type is larger than its memory type then the DAG values
3696 // are zero-extended. This breaks signed comparisons so truncate back to the
3697 // underlying type before doing the compare.
3698 if (Op1.getValueType() != MemVT) {
3699 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3700 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3701 }
3702
3703 SDNodeFlags Flags;
3704 Flags.setSameSign(I.hasSameSign());
3705 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3706
3707 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3708 I.getType());
3709 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3710}
3711
3712void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3713 FCmpInst::Predicate predicate = I.getPredicate();
3714 SDValue Op1 = getValue(I.getOperand(0));
3715 SDValue Op2 = getValue(I.getOperand(1));
3716
3717 ISD::CondCode Condition = getFCmpCondCode(predicate);
3718 auto *FPMO = cast<FPMathOperator>(&I);
3719 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3720 Condition = getFCmpCodeWithoutNaN(Condition);
3721
3722 SDNodeFlags Flags;
3723 Flags.copyFMF(*FPMO);
3724 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3725
3726 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3727 I.getType());
3728 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3729}
3730
3731// Check if the condition of the select has one use or two users that are both
3732// selects with the same condition.
3733static bool hasOnlySelectUsers(const Value *Cond) {
3734 return llvm::all_of(Cond->users(), [](const Value *V) {
3735 return isa<SelectInst>(V);
3736 });
3737}
3738
3739void SelectionDAGBuilder::visitSelect(const User &I) {
3740 SmallVector<EVT, 4> ValueVTs;
3741 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3742 ValueVTs);
3743 unsigned NumValues = ValueVTs.size();
3744 if (NumValues == 0) return;
3745
3746 SmallVector<SDValue, 4> Values(NumValues);
3747 SDValue Cond = getValue(I.getOperand(0));
3748 SDValue LHSVal = getValue(I.getOperand(1));
3749 SDValue RHSVal = getValue(I.getOperand(2));
3750 SmallVector<SDValue, 1> BaseOps(1, Cond);
3752 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3753
3754 bool IsUnaryAbs = false;
3755 bool Negate = false;
3756
3757 SDNodeFlags Flags;
3758 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3759 Flags.copyFMF(*FPOp);
3760
3761 Flags.setUnpredictable(
3762 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3763
3764 // Min/max matching is only viable if all output VTs are the same.
3765 if (all_equal(ValueVTs)) {
3766 EVT VT = ValueVTs[0];
3767 LLVMContext &Ctx = *DAG.getContext();
3768 auto &TLI = DAG.getTargetLoweringInfo();
3769
3770 // We care about the legality of the operation after it has been type
3771 // legalized.
3772 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3773 VT = TLI.getTypeToTransformTo(Ctx, VT);
3774
3775 // If the vselect is legal, assume we want to leave this as a vector setcc +
3776 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3777 // min/max is legal on the scalar type.
3778 bool UseScalarMinMax = VT.isVector() &&
3780
3781 // ValueTracking's select pattern matching does not account for -0.0,
3782 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3783 // -0.0 is less than +0.0.
3784 const Value *LHS, *RHS;
3785 auto SPR = matchSelectPattern(&I, LHS, RHS);
3787 switch (SPR.Flavor) {
3788 case SPF_UMAX: Opc = ISD::UMAX; break;
3789 case SPF_UMIN: Opc = ISD::UMIN; break;
3790 case SPF_SMAX: Opc = ISD::SMAX; break;
3791 case SPF_SMIN: Opc = ISD::SMIN; break;
3792 case SPF_FMINNUM:
3793 switch (SPR.NaNBehavior) {
3794 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3795 case SPNB_RETURNS_NAN: break;
3796 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3797 case SPNB_RETURNS_ANY:
3798 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT) ||
3799 (UseScalarMinMax &&
3800 TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType())))
3801 Opc = ISD::FMINNUM;
3802 break;
3803 }
3804 break;
3805 case SPF_FMAXNUM:
3806 switch (SPR.NaNBehavior) {
3807 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3808 case SPNB_RETURNS_NAN: break;
3809 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3810 case SPNB_RETURNS_ANY:
3811 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT) ||
3812 (UseScalarMinMax &&
3813 TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType())))
3814 Opc = ISD::FMAXNUM;
3815 break;
3816 }
3817 break;
3818 case SPF_NABS:
3819 Negate = true;
3820 [[fallthrough]];
3821 case SPF_ABS:
3822 IsUnaryAbs = true;
3823 Opc = ISD::ABS;
3824 break;
3825 default: break;
3826 }
3827
3828 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3829 (TLI.isOperationLegalOrCustom(Opc, VT) ||
3830 (UseScalarMinMax &&
3832 // If the underlying comparison instruction is used by any other
3833 // instruction, the consumed instructions won't be destroyed, so it is
3834 // not profitable to convert to a min/max.
3836 OpCode = Opc;
3837 LHSVal = getValue(LHS);
3838 RHSVal = getValue(RHS);
3839 BaseOps.clear();
3840 }
3841
3842 if (IsUnaryAbs) {
3843 OpCode = Opc;
3844 LHSVal = getValue(LHS);
3845 BaseOps.clear();
3846 }
3847 }
3848
3849 if (IsUnaryAbs) {
3850 for (unsigned i = 0; i != NumValues; ++i) {
3851 SDLoc dl = getCurSDLoc();
3852 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3853 Values[i] =
3854 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3855 if (Negate)
3856 Values[i] = DAG.getNegative(Values[i], dl, VT);
3857 }
3858 } else {
3859 for (unsigned i = 0; i != NumValues; ++i) {
3860 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3861 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3862 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3863 Values[i] = DAG.getNode(
3864 OpCode, getCurSDLoc(),
3865 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3866 }
3867 }
3868
3870 DAG.getVTList(ValueVTs), Values));
3871}
3872
3873void SelectionDAGBuilder::visitTrunc(const User &I) {
3874 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3875 SDValue N = getValue(I.getOperand(0));
3876 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3877 I.getType());
3878 SDNodeFlags Flags;
3879 if (auto *Trunc = dyn_cast<TruncInst>(&I)) {
3880 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
3881 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
3882 }
3883
3884 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N, Flags));
3885}
3886
3887void SelectionDAGBuilder::visitZExt(const User &I) {
3888 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3889 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3890 SDValue N = getValue(I.getOperand(0));
3891 auto &TLI = DAG.getTargetLoweringInfo();
3892 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3893
3894 SDNodeFlags Flags;
3895 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3896 Flags.setNonNeg(PNI->hasNonNeg());
3897
3898 // Eagerly use nonneg information to canonicalize towards sign_extend if
3899 // that is the target's preference.
3900 // TODO: Let the target do this later.
3901 if (Flags.hasNonNeg() &&
3902 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
3903 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3904 return;
3905 }
3906
3907 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
3908}
3909
3910void SelectionDAGBuilder::visitSExt(const User &I) {
3911 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3912 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3913 SDValue N = getValue(I.getOperand(0));
3914 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3915 I.getType());
3916 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3917}
3918
3919void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3920 // FPTrunc is never a no-op cast, no need to check
3921 SDValue N = getValue(I.getOperand(0));
3922 SDLoc dl = getCurSDLoc();
3923 SDNodeFlags Flags;
3924 if (auto *TruncInst = dyn_cast<FPMathOperator>(&I))
3925 Flags.copyFMF(*TruncInst);
3926 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3927 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3928 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3929 DAG.getTargetConstant(
3930 0, dl, TLI.getPointerTy(DAG.getDataLayout())),
3931 Flags));
3932}
3933
3934void SelectionDAGBuilder::visitFPExt(const User &I) {
3935 // FPExt is never a no-op cast, no need to check
3936 SDValue N = getValue(I.getOperand(0));
3937 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3938 I.getType());
3939 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3940}
3941
3942void SelectionDAGBuilder::visitFPToUI(const User &I) {
3943 // FPToUI is never a no-op cast, no need to check
3944 SDValue N = getValue(I.getOperand(0));
3945 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3946 I.getType());
3947 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3948}
3949
3950void SelectionDAGBuilder::visitFPToSI(const User &I) {
3951 // FPToSI is never a no-op cast, no need to check
3952 SDValue N = getValue(I.getOperand(0));
3953 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3954 I.getType());
3955 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3956}
3957
3958void SelectionDAGBuilder::visitUIToFP(const User &I) {
3959 // UIToFP is never a no-op cast, no need to check
3960 SDValue N = getValue(I.getOperand(0));
3961 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3962 I.getType());
3963 SDNodeFlags Flags;
3964 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3965 Flags.setNonNeg(PNI->hasNonNeg());
3966
3967 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
3968}
3969
3970void SelectionDAGBuilder::visitSIToFP(const User &I) {
3971 // SIToFP is never a no-op cast, no need to check
3972 SDValue N = getValue(I.getOperand(0));
3973 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3974 I.getType());
3975 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3976}
3977
3978void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
3979 SDValue N = getValue(I.getOperand(0));
3980 // By definition the type of the ptrtoaddr must be equal to the address type.
3981 const auto &TLI = DAG.getTargetLoweringInfo();
3982 EVT AddrVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3983 // The address width must be smaller or equal to the pointer representation
3984 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).
3985 N = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), AddrVT, N);
3986 setValue(&I, N);
3987}
3988
3989void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3990 // What to do depends on the size of the integer and the size of the pointer.
3991 // We can either truncate, zero extend, or no-op, accordingly.
3992 SDValue N = getValue(I.getOperand(0));
3993 auto &TLI = DAG.getTargetLoweringInfo();
3994 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3995 I.getType());
3996 EVT PtrMemVT =
3997 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3998 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3999 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
4000 setValue(&I, N);
4001}
4002
4003void SelectionDAGBuilder::visitIntToPtr(const User &I) {
4004 // What to do depends on the size of the integer and the size of the pointer.
4005 // We can either truncate, zero extend, or no-op, accordingly.
4006 SDValue N = getValue(I.getOperand(0));
4007 auto &TLI = DAG.getTargetLoweringInfo();
4008 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4009 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4010 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
4011 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
4012 setValue(&I, N);
4013}
4014
4015void SelectionDAGBuilder::visitBitCast(const User &I) {
4016 SDValue N = getValue(I.getOperand(0));
4017 SDLoc dl = getCurSDLoc();
4018 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4019 I.getType());
4020
4021 // BitCast assures us that source and destination are the same size so this is
4022 // either a BITCAST or a no-op.
4023 if (DestVT != N.getValueType())
4024 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
4025 DestVT, N)); // convert types.
4026 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
4027 // might fold any kind of constant expression to an integer constant and that
4028 // is not what we are looking for. Only recognize a bitcast of a genuine
4029 // constant integer as an opaque constant.
4030 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
4031 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
4032 /*isOpaque*/true));
4033 else
4034 setValue(&I, N); // noop cast.
4035}
4036
4037void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
4038 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4039 const Value *SV = I.getOperand(0);
4040 SDValue N = getValue(SV);
4041 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4042
4043 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
4044 unsigned DestAS = I.getType()->getPointerAddressSpace();
4045
4046 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
4047 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
4048
4049 setValue(&I, N);
4050}
4051
4052void SelectionDAGBuilder::visitInsertElement(const User &I) {
4053 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4054 SDValue InVec = getValue(I.getOperand(0));
4055 SDValue InVal = getValue(I.getOperand(1));
4056 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
4057 TLI.getVectorIdxTy(DAG.getDataLayout()));
4059 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4060 InVec, InVal, InIdx));
4061}
4062
4063void SelectionDAGBuilder::visitExtractElement(const User &I) {
4064 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4065 SDValue InVec = getValue(I.getOperand(0));
4066 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
4067 TLI.getVectorIdxTy(DAG.getDataLayout()));
4069 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4070 InVec, InIdx));
4071}
4072
4073void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4074 SDValue Src1 = getValue(I.getOperand(0));
4075 SDValue Src2 = getValue(I.getOperand(1));
4076 ArrayRef<int> Mask;
4077 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
4078 Mask = SVI->getShuffleMask();
4079 else
4080 Mask = cast<ConstantExpr>(I).getShuffleMask();
4081 SDLoc DL = getCurSDLoc();
4082 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4083 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4084 EVT SrcVT = Src1.getValueType();
4085
4086 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
4087 VT.isScalableVector()) {
4088 // Canonical splat form of first element of first input vector.
4089 SDValue FirstElt =
4090 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
4091 DAG.getVectorIdxConstant(0, DL));
4092 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4093 return;
4094 }
4095
4096 // For now, we only handle splats for scalable vectors.
4097 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4098 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4099 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4100
4101 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4102 unsigned MaskNumElts = Mask.size();
4103
4104 if (SrcNumElts == MaskNumElts) {
4105 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4106 return;
4107 }
4108
4109 // Normalize the shuffle vector since mask and vector length don't match.
4110 if (SrcNumElts < MaskNumElts) {
4111 // Mask is longer than the source vectors. We can use concatenate vector to
4112 // make the mask and vectors lengths match.
4113
4114 if (MaskNumElts % SrcNumElts == 0) {
4115 // Mask length is a multiple of the source vector length.
4116 // Check if the shuffle is some kind of concatenation of the input
4117 // vectors.
4118 unsigned NumConcat = MaskNumElts / SrcNumElts;
4119 bool IsConcat = true;
4120 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4121 for (unsigned i = 0; i != MaskNumElts; ++i) {
4122 int Idx = Mask[i];
4123 if (Idx < 0)
4124 continue;
4125 // Ensure the indices in each SrcVT sized piece are sequential and that
4126 // the same source is used for the whole piece.
4127 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4128 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4129 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4130 IsConcat = false;
4131 break;
4132 }
4133 // Remember which source this index came from.
4134 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4135 }
4136
4137 // The shuffle is concatenating multiple vectors together. Just emit
4138 // a CONCAT_VECTORS operation.
4139 if (IsConcat) {
4140 SmallVector<SDValue, 8> ConcatOps;
4141 for (auto Src : ConcatSrcs) {
4142 if (Src < 0)
4143 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4144 else if (Src == 0)
4145 ConcatOps.push_back(Src1);
4146 else
4147 ConcatOps.push_back(Src2);
4148 }
4149 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4150 return;
4151 }
4152 }
4153
4154 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4155 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4156 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4157 PaddedMaskNumElts);
4158
4159 // Pad both vectors with undefs to make them the same length as the mask.
4160 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4161
4162 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4163 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4164 MOps1[0] = Src1;
4165 MOps2[0] = Src2;
4166
4167 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4168 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4169
4170 // Readjust mask for new input vector length.
4171 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4172 for (unsigned i = 0; i != MaskNumElts; ++i) {
4173 int Idx = Mask[i];
4174 if (Idx >= (int)SrcNumElts)
4175 Idx -= SrcNumElts - PaddedMaskNumElts;
4176 MappedOps[i] = Idx;
4177 }
4178
4179 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4180
4181 // If the concatenated vector was padded, extract a subvector with the
4182 // correct number of elements.
4183 if (MaskNumElts != PaddedMaskNumElts)
4184 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
4185 DAG.getVectorIdxConstant(0, DL));
4186
4187 setValue(&I, Result);
4188 return;
4189 }
4190
4191 assert(SrcNumElts > MaskNumElts);
4192
4193 // Analyze the access pattern of the vector to see if we can extract
4194 // two subvectors and do the shuffle.
4195 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4196 bool CanExtract = true;
4197 for (int Idx : Mask) {
4198 unsigned Input = 0;
4199 if (Idx < 0)
4200 continue;
4201
4202 if (Idx >= (int)SrcNumElts) {
4203 Input = 1;
4204 Idx -= SrcNumElts;
4205 }
4206
4207 // If all the indices come from the same MaskNumElts sized portion of
4208 // the sources we can use extract. Also make sure the extract wouldn't
4209 // extract past the end of the source.
4210 int NewStartIdx = alignDown(Idx, MaskNumElts);
4211 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4212 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4213 CanExtract = false;
4214 // Make sure we always update StartIdx as we use it to track if all
4215 // elements are undef.
4216 StartIdx[Input] = NewStartIdx;
4217 }
4218
4219 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4220 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4221 return;
4222 }
4223 if (CanExtract) {
4224 // Extract appropriate subvector and generate a vector shuffle
4225 for (unsigned Input = 0; Input < 2; ++Input) {
4226 SDValue &Src = Input == 0 ? Src1 : Src2;
4227 if (StartIdx[Input] < 0)
4228 Src = DAG.getUNDEF(VT);
4229 else {
4230 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4231 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4232 }
4233 }
4234
4235 // Calculate new mask.
4236 SmallVector<int, 8> MappedOps(Mask);
4237 for (int &Idx : MappedOps) {
4238 if (Idx >= (int)SrcNumElts)
4239 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4240 else if (Idx >= 0)
4241 Idx -= StartIdx[0];
4242 }
4243
4244 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4245 return;
4246 }
4247
4248 // We can't use either concat vectors or extract subvectors so fall back to
4249 // replacing the shuffle with extract and build vector.
4250 // to insert and build vector.
4251 EVT EltVT = VT.getVectorElementType();
4253 for (int Idx : Mask) {
4254 SDValue Res;
4255
4256 if (Idx < 0) {
4257 Res = DAG.getUNDEF(EltVT);
4258 } else {
4259 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4260 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4261
4262 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4263 DAG.getVectorIdxConstant(Idx, DL));
4264 }
4265
4266 Ops.push_back(Res);
4267 }
4268
4269 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4270}
4271
4272void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4273 ArrayRef<unsigned> Indices = I.getIndices();
4274 const Value *Op0 = I.getOperand(0);
4275 const Value *Op1 = I.getOperand(1);
4276 Type *AggTy = I.getType();
4277 Type *ValTy = Op1->getType();
4278 bool IntoUndef = isa<UndefValue>(Op0);
4279 bool FromUndef = isa<UndefValue>(Op1);
4280
4281 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4282
4283 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4284 SmallVector<EVT, 4> AggValueVTs;
4285 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4286 SmallVector<EVT, 4> ValValueVTs;
4287 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4288
4289 unsigned NumAggValues = AggValueVTs.size();
4290 unsigned NumValValues = ValValueVTs.size();
4291 SmallVector<SDValue, 4> Values(NumAggValues);
4292
4293 // Ignore an insertvalue that produces an empty object
4294 if (!NumAggValues) {
4295 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4296 return;
4297 }
4298
4299 SDValue Agg = getValue(Op0);
4300 unsigned i = 0;
4301 // Copy the beginning value(s) from the original aggregate.
4302 for (; i != LinearIndex; ++i)
4303 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4304 SDValue(Agg.getNode(), Agg.getResNo() + i);
4305 // Copy values from the inserted value(s).
4306 if (NumValValues) {
4307 SDValue Val = getValue(Op1);
4308 for (; i != LinearIndex + NumValValues; ++i)
4309 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4310 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4311 }
4312 // Copy remaining value(s) from the original aggregate.
4313 for (; i != NumAggValues; ++i)
4314 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4315 SDValue(Agg.getNode(), Agg.getResNo() + i);
4316
4318 DAG.getVTList(AggValueVTs), Values));
4319}
4320
4321void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4322 ArrayRef<unsigned> Indices = I.getIndices();
4323 const Value *Op0 = I.getOperand(0);
4324 Type *AggTy = Op0->getType();
4325 Type *ValTy = I.getType();
4326 bool OutOfUndef = isa<UndefValue>(Op0);
4327
4328 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4329
4330 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4331 SmallVector<EVT, 4> ValValueVTs;
4332 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4333
4334 unsigned NumValValues = ValValueVTs.size();
4335
4336 // Ignore a extractvalue that produces an empty object
4337 if (!NumValValues) {
4338 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4339 return;
4340 }
4341
4342 SmallVector<SDValue, 4> Values(NumValValues);
4343
4344 SDValue Agg = getValue(Op0);
4345 // Copy out the selected value(s).
4346 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4347 Values[i - LinearIndex] =
4348 OutOfUndef ?
4349 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4350 SDValue(Agg.getNode(), Agg.getResNo() + i);
4351
4353 DAG.getVTList(ValValueVTs), Values));
4354}
4355
4356void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4357 Value *Op0 = I.getOperand(0);
4358 // Note that the pointer operand may be a vector of pointers. Take the scalar
4359 // element which holds a pointer.
4360 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4361 SDValue N = getValue(Op0);
4362 SDLoc dl = getCurSDLoc();
4363 auto &TLI = DAG.getTargetLoweringInfo();
4364 GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();
4365
4366 // For a vector GEP, keep the prefix scalar as long as possible, then
4367 // convert any scalars encountered after the first vector operand to vectors.
4368 bool IsVectorGEP = I.getType()->isVectorTy();
4369 ElementCount VectorElementCount =
4370 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4372
4374 GTI != E; ++GTI) {
4375 const Value *Idx = GTI.getOperand();
4376 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4377 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4378 if (Field) {
4379 // N = N + Offset
4380 uint64_t Offset =
4381 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
4382
4383 // In an inbounds GEP with an offset that is nonnegative even when
4384 // interpreted as signed, assume there is no unsigned overflow.
4385 SDNodeFlags Flags;
4386 if (NW.hasNoUnsignedWrap() ||
4387 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4389
4390 N = DAG.getMemBasePlusOffset(
4391 N, DAG.getConstant(Offset, dl, N.getValueType()), dl, Flags);
4392 }
4393 } else {
4394 // IdxSize is the width of the arithmetic according to IR semantics.
4395 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4396 // (and fix up the result later).
4397 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4398 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4399 TypeSize ElementSize =
4400 GTI.getSequentialElementStride(DAG.getDataLayout());
4401 // We intentionally mask away the high bits here; ElementSize may not
4402 // fit in IdxTy.
4403 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4404 /*isSigned=*/false, /*implicitTrunc=*/true);
4405 bool ElementScalable = ElementSize.isScalable();
4406
4407 // If this is a scalar constant or a splat vector of constants,
4408 // handle it quickly.
4409 const auto *C = dyn_cast<Constant>(Idx);
4410 if (C && isa<VectorType>(C->getType()))
4411 C = C->getSplatValue();
4412
4413 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4414 if (CI && CI->isZero())
4415 continue;
4416 if (CI && !ElementScalable) {
4417 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4418 LLVMContext &Context = *DAG.getContext();
4419 SDValue OffsVal;
4420 if (N.getValueType().isVector())
4421 OffsVal = DAG.getConstant(
4422 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4423 else
4424 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4425
4426 // In an inbounds GEP with an offset that is nonnegative even when
4427 // interpreted as signed, assume there is no unsigned overflow.
4428 SDNodeFlags Flags;
4429 if (NW.hasNoUnsignedWrap() ||
4430 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4431 Flags.setNoUnsignedWrap(true);
4432
4433 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4434
4435 N = DAG.getMemBasePlusOffset(N, OffsVal, dl, Flags);
4436 continue;
4437 }
4438
4439 // N = N + Idx * ElementMul;
4440 SDValue IdxN = getValue(Idx);
4441
4442 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
4443 if (N.getValueType().isVector()) {
4444 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
4445 VectorElementCount);
4446 IdxN = DAG.getSplat(VT, dl, IdxN);
4447 } else {
4448 EVT VT =
4449 EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
4450 N = DAG.getSplat(VT, dl, N);
4451 }
4452 }
4453
4454 // If the index is smaller or larger than intptr_t, truncate or extend
4455 // it.
4456 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4457
4458 SDNodeFlags ScaleFlags;
4459 // The multiplication of an index by the type size does not wrap the
4460 // pointer index type in a signed sense (mul nsw).
4462
4463 // The multiplication of an index by the type size does not wrap the
4464 // pointer index type in an unsigned sense (mul nuw).
4465 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4466
4467 if (ElementScalable) {
4468 EVT VScaleTy = N.getValueType().getScalarType();
4469 SDValue VScale = DAG.getNode(
4470 ISD::VSCALE, dl, VScaleTy,
4471 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4472 if (N.getValueType().isVector())
4473 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4474 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale,
4475 ScaleFlags);
4476 } else {
4477 // If this is a multiply by a power of two, turn it into a shl
4478 // immediately. This is a very common case.
4479 if (ElementMul != 1) {
4480 if (ElementMul.isPowerOf2()) {
4481 unsigned Amt = ElementMul.logBase2();
4482 IdxN = DAG.getNode(
4483 ISD::SHL, dl, N.getValueType(), IdxN,
4484 DAG.getShiftAmountConstant(Amt, N.getValueType(), dl),
4485 ScaleFlags);
4486 } else {
4487 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4488 IdxN.getValueType());
4489 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, Scale,
4490 ScaleFlags);
4491 }
4492 }
4493 }
4494
4495 // The successive addition of the current address, truncated to the
4496 // pointer index type and interpreted as an unsigned number, and each
4497 // offset, also interpreted as an unsigned number, does not wrap the
4498 // pointer index type (add nuw).
4499 SDNodeFlags AddFlags;
4500 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4501
4502 N = DAG.getMemBasePlusOffset(N, IdxN, dl, AddFlags);
4503 }
4504 }
4505
4506 if (IsVectorGEP && !N.getValueType().isVector()) {
4507 EVT VT = EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
4508 N = DAG.getSplat(VT, dl, N);
4509 }
4510
4511 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4512 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4513 if (IsVectorGEP) {
4514 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4515 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4516 }
4517
4518 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4519 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4520
4521 setValue(&I, N);
4522}
4523
4524void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4525 // If this is a fixed sized alloca in the entry block of the function,
4526 // allocate it statically on the stack.
4527 if (FuncInfo.StaticAllocaMap.count(&I))
4528 return; // getValue will auto-populate this.
4529
4530 SDLoc dl = getCurSDLoc();
4531 Type *Ty = I.getAllocatedType();
4532 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4533 auto &DL = DAG.getDataLayout();
4534 TypeSize TySize = DL.getTypeAllocSize(Ty);
4535 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4536
4537 SDValue AllocSize = getValue(I.getArraySize());
4538
4539 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4540 if (AllocSize.getValueType() != IntPtr)
4541 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4542
4543 if (TySize.isScalable())
4544 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4545 DAG.getVScale(dl, IntPtr,
4546 APInt(IntPtr.getScalarSizeInBits(),
4547 TySize.getKnownMinValue())));
4548 else {
4549 SDValue TySizeValue =
4550 DAG.getConstant(TySize.getFixedValue(), dl, MVT::getIntegerVT(64));
4551 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4552 DAG.getZExtOrTrunc(TySizeValue, dl, IntPtr));
4553 }
4554
4555 // Handle alignment. If the requested alignment is less than or equal to
4556 // the stack alignment, ignore it. If the size is greater than or equal to
4557 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4558 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4559 if (*Alignment <= StackAlign)
4560 Alignment = std::nullopt;
4561
4562 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4563 // Round the size of the allocation up to the stack alignment size
4564 // by add SA-1 to the size. This doesn't overflow because we're computing
4565 // an address inside an alloca.
4566 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4567 DAG.getConstant(StackAlignMask, dl, IntPtr),
4569
4570 // Mask out the low bits for alignment purposes.
4571 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4572 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
4573
4574 SDValue Ops[] = {
4575 getRoot(), AllocSize,
4576 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4577 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4578 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4579 setValue(&I, DSA);
4580 DAG.setRoot(DSA.getValue(1));
4581
4582 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4583}
4584
4585static const MDNode *getRangeMetadata(const Instruction &I) {
4586 return I.getMetadata(LLVMContext::MD_range);
4587}
4588
4589static std::optional<ConstantRange> getRange(const Instruction &I) {
4590 if (const auto *CB = dyn_cast<CallBase>(&I))
4591 if (std::optional<ConstantRange> CR = CB->getRange())
4592 return CR;
4593 if (const MDNode *Range = getRangeMetadata(I))
4595 return std::nullopt;
4596}
4597
4598void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4599 if (I.isAtomic())
4600 return visitAtomicLoad(I);
4601
4602 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4603 const Value *SV = I.getOperand(0);
4604 if (TLI.supportSwiftError()) {
4605 // Swifterror values can come from either a function parameter with
4606 // swifterror attribute or an alloca with swifterror attribute.
4607 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4608 if (Arg->hasSwiftErrorAttr())
4609 return visitLoadFromSwiftError(I);
4610 }
4611
4612 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4613 if (Alloca->isSwiftError())
4614 return visitLoadFromSwiftError(I);
4615 }
4616 }
4617
4618 SDValue Ptr = getValue(SV);
4619
4620 Type *Ty = I.getType();
4621 SmallVector<EVT, 4> ValueVTs, MemVTs;
4623 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4624 unsigned NumValues = ValueVTs.size();
4625 if (NumValues == 0)
4626 return;
4627
4628 Align Alignment = I.getAlign();
4629 AAMDNodes AAInfo = I.getAAMetadata();
4630 const MDNode *Ranges = getRangeMetadata(I);
4631 bool isVolatile = I.isVolatile();
4632 MachineMemOperand::Flags MMOFlags =
4633 TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
4634
4635 SDValue Root;
4636 bool ConstantMemory = false;
4637 if (isVolatile)
4638 // Serialize volatile loads with other side effects.
4639 Root = getRoot();
4640 else if (NumValues > MaxParallelChains)
4641 Root = getMemoryRoot();
4642 else if (BatchAA &&
4643 BatchAA->pointsToConstantMemory(MemoryLocation(
4644 SV,
4645 LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4646 AAInfo))) {
4647 // Do not serialize (non-volatile) loads of constant memory with anything.
4648 Root = DAG.getEntryNode();
4649 ConstantMemory = true;
4651 } else {
4652 // Do not serialize non-volatile loads against each other.
4653 Root = DAG.getRoot();
4654 }
4655
4656 SDLoc dl = getCurSDLoc();
4657
4658 if (isVolatile)
4659 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4660
4661 SmallVector<SDValue, 4> Values(NumValues);
4662 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4663
4664 unsigned ChainI = 0;
4665 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4666 // Serializing loads here may result in excessive register pressure, and
4667 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4668 // could recover a bit by hoisting nodes upward in the chain by recognizing
4669 // they are side-effect free or do not alias. The optimizer should really
4670 // avoid this case by converting large object/array copies to llvm.memcpy
4671 // (MaxParallelChains should always remain as failsafe).
4672 if (ChainI == MaxParallelChains) {
4673 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4674 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4675 ArrayRef(Chains.data(), ChainI));
4676 Root = Chain;
4677 ChainI = 0;
4678 }
4679
4680 // TODO: MachinePointerInfo only supports a fixed length offset.
4681 MachinePointerInfo PtrInfo =
4682 !Offsets[i].isScalable() || Offsets[i].isZero()
4683 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4684 : MachinePointerInfo();
4685
4686 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4687 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
4688 MMOFlags, AAInfo, Ranges);
4689 Chains[ChainI] = L.getValue(1);
4690
4691 if (MemVTs[i] != ValueVTs[i])
4692 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4693
4694 Values[i] = L;
4695 }
4696
4697 if (!ConstantMemory) {
4698 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4699 ArrayRef(Chains.data(), ChainI));
4700 if (isVolatile)
4701 DAG.setRoot(Chain);
4702 else
4703 PendingLoads.push_back(Chain);
4704 }
4705
4706 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4707 DAG.getVTList(ValueVTs), Values));
4708}
4709
4710void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4711 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4712 "call visitStoreToSwiftError when backend supports swifterror");
4713
4714 SmallVector<EVT, 4> ValueVTs;
4715 SmallVector<uint64_t, 4> Offsets;
4716 const Value *SrcV = I.getOperand(0);
4717 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4718 SrcV->getType(), ValueVTs, &Offsets, 0);
4719 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4720 "expect a single EVT for swifterror");
4721
4722 SDValue Src = getValue(SrcV);
4723 // Create a virtual register, then update the virtual register.
4724 Register VReg =
4725 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4726 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4727 // Chain can be getRoot or getControlRoot.
4728 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4729 SDValue(Src.getNode(), Src.getResNo()));
4730 DAG.setRoot(CopyNode);
4731}
4732
4733void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4734 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4735 "call visitLoadFromSwiftError when backend supports swifterror");
4736
4737 assert(!I.isVolatile() &&
4738 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4739 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4740 "Support volatile, non temporal, invariant for load_from_swift_error");
4741
4742 const Value *SV = I.getOperand(0);
4743 Type *Ty = I.getType();
4744 assert(
4745 (!BatchAA ||
4746 !BatchAA->pointsToConstantMemory(MemoryLocation(
4747 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4748 I.getAAMetadata()))) &&
4749 "load_from_swift_error should not be constant memory");
4750
4751 SmallVector<EVT, 4> ValueVTs;
4752 SmallVector<uint64_t, 4> Offsets;
4753 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
4754 ValueVTs, &Offsets, 0);
4755 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4756 "expect a single EVT for swifterror");
4757
4758 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4759 SDValue L = DAG.getCopyFromReg(
4760 getRoot(), getCurSDLoc(),
4761 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4762
4763 setValue(&I, L);
4764}
4765
4766void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4767 if (I.isAtomic())
4768 return visitAtomicStore(I);
4769
4770 const Value *SrcV = I.getOperand(0);
4771 const Value *PtrV = I.getOperand(1);
4772
4773 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4774 if (TLI.supportSwiftError()) {
4775 // Swifterror values can come from either a function parameter with
4776 // swifterror attribute or an alloca with swifterror attribute.
4777 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4778 if (Arg->hasSwiftErrorAttr())
4779 return visitStoreToSwiftError(I);
4780 }
4781
4782 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4783 if (Alloca->isSwiftError())
4784 return visitStoreToSwiftError(I);
4785 }
4786 }
4787
4788 SmallVector<EVT, 4> ValueVTs, MemVTs;
4790 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4791 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4792 unsigned NumValues = ValueVTs.size();
4793 if (NumValues == 0)
4794 return;
4795
4796 // Get the lowered operands. Note that we do this after
4797 // checking if NumResults is zero, because with zero results
4798 // the operands won't have values in the map.
4799 SDValue Src = getValue(SrcV);
4800 SDValue Ptr = getValue(PtrV);
4801
4802 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4803 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4804 SDLoc dl = getCurSDLoc();
4805 Align Alignment = I.getAlign();
4806 AAMDNodes AAInfo = I.getAAMetadata();
4807
4808 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4809
4810 unsigned ChainI = 0;
4811 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4812 // See visitLoad comments.
4813 if (ChainI == MaxParallelChains) {
4814 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4815 ArrayRef(Chains.data(), ChainI));
4816 Root = Chain;
4817 ChainI = 0;
4818 }
4819
4820 // TODO: MachinePointerInfo only supports a fixed length offset.
4821 MachinePointerInfo PtrInfo =
4822 !Offsets[i].isScalable() || Offsets[i].isZero()
4823 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4824 : MachinePointerInfo();
4825
4826 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4827 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4828 if (MemVTs[i] != ValueVTs[i])
4829 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4830 SDValue St =
4831 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4832 Chains[ChainI] = St;
4833 }
4834
4835 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4836 ArrayRef(Chains.data(), ChainI));
4837 setValue(&I, StoreNode);
4838 DAG.setRoot(StoreNode);
4839}
4840
4841void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4842 bool IsCompressing) {
4843 SDLoc sdl = getCurSDLoc();
4844
4845 Value *Src0Operand = I.getArgOperand(0);
4846 Value *PtrOperand = I.getArgOperand(1);
4847 Value *MaskOperand = I.getArgOperand(2);
4848 Align Alignment = I.getParamAlign(1).valueOrOne();
4849
4850 SDValue Ptr = getValue(PtrOperand);
4851 SDValue Src0 = getValue(Src0Operand);
4852 SDValue Mask = getValue(MaskOperand);
4853 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4854
4855 EVT VT = Src0.getValueType();
4856
4857 auto MMOFlags = MachineMemOperand::MOStore;
4858 if (I.hasMetadata(LLVMContext::MD_nontemporal))
4860
4861 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4862 MachinePointerInfo(PtrOperand), MMOFlags,
4863 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4864
4865 const auto &TLI = DAG.getTargetLoweringInfo();
4866 const auto &TTI =
4867 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
4868 SDValue StoreNode =
4869 !IsCompressing && TTI.hasConditionalLoadStoreForType(
4870 I.getArgOperand(0)->getType(), /*IsStore=*/true)
4871 ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0,
4872 Mask)
4873 : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask,
4874 VT, MMO, ISD::UNINDEXED, /*Truncating=*/false,
4875 IsCompressing);
4876 DAG.setRoot(StoreNode);
4877 setValue(&I, StoreNode);
4878}
4879
4880// Get a uniform base for the Gather/Scatter intrinsic.
4881// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4882// We try to represent it as a base pointer + vector of indices.
4883// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4884// The first operand of the GEP may be a single pointer or a vector of pointers
4885// Example:
4886// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4887// or
4888// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4889// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4890//
4891// When the first GEP operand is a single pointer - it is the uniform base we
4892// are looking for. If first operand of the GEP is a splat vector - we
4893// extract the splat value and use it as a uniform base.
4894// In all other cases the function returns 'false'.
4895static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4896 SDValue &Scale, SelectionDAGBuilder *SDB,
4897 const BasicBlock *CurBB, uint64_t ElemSize) {
4898 SelectionDAG& DAG = SDB->DAG;
4899 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4900 const DataLayout &DL = DAG.getDataLayout();
4901
4902 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4903
4904 // Handle splat constant pointer.
4905 if (auto *C = dyn_cast<Constant>(Ptr)) {
4906 C = C->getSplatValue();
4907 if (!C)
4908 return false;
4909
4910 Base = SDB->getValue(C);
4911
4912 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4913 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4914 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4915 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4916 return true;
4917 }
4918
4920 if (!GEP || GEP->getParent() != CurBB)
4921 return false;
4922
4923 if (GEP->getNumOperands() != 2)
4924 return false;
4925
4926 const Value *BasePtr = GEP->getPointerOperand();
4927 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4928
4929 // Make sure the base is scalar and the index is a vector.
4930 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4931 return false;
4932
4933 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4934 if (ScaleVal.isScalable())
4935 return false;
4936
4937 // Target may not support the required addressing mode.
4938 if (ScaleVal != 1 &&
4939 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
4940 return false;
4941
4942 Base = SDB->getValue(BasePtr);
4943 Index = SDB->getValue(IndexVal);
4944
4945 Scale =
4946 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4947 return true;
4948}
4949
4950void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4951 SDLoc sdl = getCurSDLoc();
4952
4953 // llvm.masked.scatter.*(Src0, Ptrs, Mask)
4954 const Value *Ptr = I.getArgOperand(1);
4955 SDValue Src0 = getValue(I.getArgOperand(0));
4956 SDValue Mask = getValue(I.getArgOperand(2));
4957 EVT VT = Src0.getValueType();
4958 Align Alignment = I.getParamAlign(1).valueOrOne();
4959 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4960
4961 SDValue Base;
4962 SDValue Index;
4963 SDValue Scale;
4964 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
4965 I.getParent(), VT.getScalarStoreSize());
4966
4967 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4968 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4969 MachinePointerInfo(AS), MachineMemOperand::MOStore,
4970 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4971 if (!UniformBase) {
4972 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4973 Index = getValue(Ptr);
4974 Scale =
4975 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4976 }
4977
4978 EVT IdxVT = Index.getValueType();
4979 EVT EltTy = IdxVT.getVectorElementType();
4980 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4981 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4982 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4983 }
4984
4985 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4986 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4987 Ops, MMO, ISD::SIGNED_SCALED, false);
4988 DAG.setRoot(Scatter);
4989 setValue(&I, Scatter);
4990}
4991
4992void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4993 SDLoc sdl = getCurSDLoc();
4994
4995 Value *PtrOperand = I.getArgOperand(0);
4996 Value *MaskOperand = I.getArgOperand(1);
4997 Value *Src0Operand = I.getArgOperand(2);
4998 Align Alignment = I.getParamAlign(0).valueOrOne();
4999
5000 SDValue Ptr = getValue(PtrOperand);
5001 SDValue Src0 = getValue(Src0Operand);
5002 SDValue Mask = getValue(MaskOperand);
5003 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
5004
5005 EVT VT = Src0.getValueType();
5006 AAMDNodes AAInfo = I.getAAMetadata();
5007 const MDNode *Ranges = getRangeMetadata(I);
5008
5009 // Do not serialize masked loads of constant memory with anything.
5010 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
5011 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
5012
5013 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5014
5015 auto MMOFlags = MachineMemOperand::MOLoad;
5016 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5018
5019 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5020 MachinePointerInfo(PtrOperand), MMOFlags,
5021 LocationSize::beforeOrAfterPointer(), Alignment, AAInfo, Ranges);
5022
5023 const auto &TLI = DAG.getTargetLoweringInfo();
5024 const auto &TTI =
5025 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
5026 // The Load/Res may point to different values and both of them are output
5027 // variables.
5028 SDValue Load;
5029 SDValue Res;
5030 if (!IsExpanding && TTI.hasConditionalLoadStoreForType(Src0Operand->getType(),
5031 /*IsStore=*/false))
5032 Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask);
5033 else
5034 Res = Load =
5035 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
5036 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5037 if (AddToChain)
5038 PendingLoads.push_back(Load.getValue(1));
5039 setValue(&I, Res);
5040}
5041
5042void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5043 SDLoc sdl = getCurSDLoc();
5044
5045 // @llvm.masked.gather.*(Ptrs, Mask, Src0)
5046 const Value *Ptr = I.getArgOperand(0);
5047 SDValue Src0 = getValue(I.getArgOperand(2));
5048 SDValue Mask = getValue(I.getArgOperand(1));
5049
5050 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5051 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5052 Align Alignment = I.getParamAlign(0).valueOrOne();
5053
5054 const MDNode *Ranges = getRangeMetadata(I);
5055
5056 SDValue Root = DAG.getRoot();
5057 SDValue Base;
5058 SDValue Index;
5059 SDValue Scale;
5060 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
5061 I.getParent(), VT.getScalarStoreSize());
5062 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5063 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5064 MachinePointerInfo(AS), MachineMemOperand::MOLoad,
5065 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
5066 Ranges);
5067
5068 if (!UniformBase) {
5069 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5070 Index = getValue(Ptr);
5071 Scale =
5072 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5073 }
5074
5075 EVT IdxVT = Index.getValueType();
5076 EVT EltTy = IdxVT.getVectorElementType();
5077 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5078 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
5079 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5080 }
5081
5082 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5083 SDValue Gather =
5084 DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, Ops, MMO,
5086
5087 PendingLoads.push_back(Gather.getValue(1));
5088 setValue(&I, Gather);
5089}
5090
5091void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5092 SDLoc dl = getCurSDLoc();
5093 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5094 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5095 SyncScope::ID SSID = I.getSyncScopeID();
5096
5097 SDValue InChain = getRoot();
5098
5099 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
5100 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5101
5102 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5103 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
5104
5105 MachineFunction &MF = DAG.getMachineFunction();
5106 MachineMemOperand *MMO = MF.getMachineMemOperand(
5107 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5108 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
5109 FailureOrdering);
5110
5111 SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
5112 dl, MemVT, VTs, InChain,
5113 getValue(I.getPointerOperand()),
5114 getValue(I.getCompareOperand()),
5115 getValue(I.getNewValOperand()), MMO);
5116
5117 SDValue OutChain = L.getValue(2);
5118
5119 setValue(&I, L);
5120 DAG.setRoot(OutChain);
5121}
5122
5123void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5124 SDLoc dl = getCurSDLoc();
5126 switch (I.getOperation()) {
5127 default: llvm_unreachable("Unknown atomicrmw operation");
5128 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
5129 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
5130 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
5131 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
5132 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
5133 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
5134 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
5135 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
5136 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
5137 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
5138 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
5139 case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
5140 case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
5141 case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
5142 case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
5144 NT = ISD::ATOMIC_LOAD_FMAXIMUM;
5145 break;
5147 NT = ISD::ATOMIC_LOAD_FMINIMUM;
5148 break;
5150 NT = ISD::ATOMIC_LOAD_UINC_WRAP;
5151 break;
5153 NT = ISD::ATOMIC_LOAD_UDEC_WRAP;
5154 break;
5156 NT = ISD::ATOMIC_LOAD_USUB_COND;
5157 break;
5159 NT = ISD::ATOMIC_LOAD_USUB_SAT;
5160 break;
5161 }
5162 AtomicOrdering Ordering = I.getOrdering();
5163 SyncScope::ID SSID = I.getSyncScopeID();
5164
5165 SDValue InChain = getRoot();
5166
5167 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5168 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5169 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
5170
5171 MachineFunction &MF = DAG.getMachineFunction();
5172 MachineMemOperand *MMO = MF.getMachineMemOperand(
5173 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5174 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
5175
5176 SDValue L =
5177 DAG.getAtomic(NT, dl, MemVT, InChain,
5178 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5179 MMO);
5180
5181 SDValue OutChain = L.getValue(1);
5182
5183 setValue(&I, L);
5184 DAG.setRoot(OutChain);
5185}
5186
5187void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5188 SDLoc dl = getCurSDLoc();
5189 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5190 SDValue Ops[3];
5191 Ops[0] = getRoot();
5192 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5193 TLI.getFenceOperandTy(DAG.getDataLayout()));
5194 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5195 TLI.getFenceOperandTy(DAG.getDataLayout()));
5196 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5197 setValue(&I, N);
5198 DAG.setRoot(N);
5199}
5200
5201void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5202 SDLoc dl = getCurSDLoc();
5203 AtomicOrdering Order = I.getOrdering();
5204 SyncScope::ID SSID = I.getSyncScopeID();
5205
5206 SDValue InChain = getRoot();
5207
5208 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5209 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5210 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5211
5212 if (!TLI.supportsUnalignedAtomics() &&
5213 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5214 report_fatal_error("Cannot generate unaligned atomic load");
5215
5216 auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
5217
5218 const MDNode *Ranges = getRangeMetadata(I);
5219 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5220 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5221 I.getAlign(), AAMDNodes(), Ranges, SSID, Order);
5222
5223 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5224
5225 SDValue Ptr = getValue(I.getPointerOperand());
5226 SDValue L =
5227 DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, MemVT, MemVT, InChain, Ptr, MMO);
5228
5229 SDValue OutChain = L.getValue(1);
5230 if (MemVT != VT)
5231 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5232
5233 setValue(&I, L);
5234 DAG.setRoot(OutChain);
5235}
5236
5237void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5238 SDLoc dl = getCurSDLoc();
5239
5240 AtomicOrdering Ordering = I.getOrdering();
5241 SyncScope::ID SSID = I.getSyncScopeID();
5242
5243 SDValue InChain = getRoot();
5244
5245 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5246 EVT MemVT =
5247 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5248
5249 if (!TLI.supportsUnalignedAtomics() &&
5250 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5251 report_fatal_error("Cannot generate unaligned atomic store");
5252
5253 auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
5254
5255 MachineFunction &MF = DAG.getMachineFunction();
5256 MachineMemOperand *MMO = MF.getMachineMemOperand(
5257 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5258 I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
5259
5260 SDValue Val = getValue(I.getValueOperand());
5261 if (Val.getValueType() != MemVT)
5262 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5263 SDValue Ptr = getValue(I.getPointerOperand());
5264
5265 SDValue OutChain =
5266 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5267
5268 setValue(&I, OutChain);
5269 DAG.setRoot(OutChain);
5270}
5271
5272/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5273/// node.
5274void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5275 unsigned Intrinsic) {
5276 // Ignore the callsite's attributes. A specific call site may be marked with
5277 // readnone, but the lowering code will expect the chain based on the
5278 // definition.
5279 const Function *F = I.getCalledFunction();
5280 bool HasChain = !F->doesNotAccessMemory();
5281 bool OnlyLoad =
5282 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5283
5284 // Build the operand list.
5286 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5287 if (OnlyLoad) {
5288 // We don't need to serialize loads against other loads.
5289 Ops.push_back(DAG.getRoot());
5290 } else {
5291 Ops.push_back(getRoot());
5292 }
5293 }
5294
5295 // Info is set by getTgtMemIntrinsic
5296 TargetLowering::IntrinsicInfo Info;
5297 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5298 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
5299 DAG.getMachineFunction(),
5300 Intrinsic);
5301
5302 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5303 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
5305 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
5306 TLI.getPointerTy(DAG.getDataLayout())));
5307
5308 // Add all operands of the call to the operand list.
5309 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5310 const Value *Arg = I.getArgOperand(i);
5311 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5312 Ops.push_back(getValue(Arg));
5313 continue;
5314 }
5315
5316 // Use TargetConstant instead of a regular constant for immarg.
5317 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5318 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5319 assert(CI->getBitWidth() <= 64 &&
5320 "large intrinsic immediates not handled");
5321 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5322 } else {
5323 Ops.push_back(
5324 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5325 }
5326 }
5327
5328 SmallVector<EVT, 4> ValueVTs;
5329 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5330
5331 if (HasChain)
5332 ValueVTs.push_back(MVT::Other);
5333
5334 SDVTList VTs = DAG.getVTList(ValueVTs);
5335
5336 // Propagate fast-math-flags from IR to node(s).
5337 SDNodeFlags Flags;
5338 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5339 Flags.copyFMF(*FPMO);
5340 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5341
5342 // Create the node.
5344
5345 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5346 auto *Token = Bundle->Inputs[0].get();
5347 SDValue ConvControlToken = getValue(Token);
5348 assert(Ops.back().getValueType() != MVT::Glue &&
5349 "Did not expected another glue node here.");
5350 ConvControlToken =
5351 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5352 Ops.push_back(ConvControlToken);
5353 }
5354
5355 // In some cases, custom collection of operands from CallInst I may be needed.
5357 if (IsTgtIntrinsic) {
5358 // This is target intrinsic that touches memory
5359 //
5360 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
5361 // didn't yield anything useful.
5362 MachinePointerInfo MPI;
5363 if (Info.ptrVal)
5364 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5365 else if (Info.fallbackAddressSpace)
5366 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5367 EVT MemVT = Info.memVT;
5368 LocationSize Size = LocationSize::precise(Info.size);
5369 if (Size.hasValue() && !Size.getValue())
5371 Align Alignment = Info.align.value_or(DAG.getEVTAlign(MemVT));
5372 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5373 MPI, Info.flags, Size, Alignment, I.getAAMetadata(), /*Ranges=*/nullptr,
5374 Info.ssid, Info.order, Info.failureOrder);
5375 Result =
5376 DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, MemVT, MMO);
5377 } else if (!HasChain) {
5378 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
5379 } else if (!I.getType()->isVoidTy()) {
5380 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
5381 } else {
5382 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
5383 }
5384
5385 if (HasChain) {
5386 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
5387 if (OnlyLoad)
5388 PendingLoads.push_back(Chain);
5389 else
5390 DAG.setRoot(Chain);
5391 }
5392
5393 if (!I.getType()->isVoidTy()) {
5394 if (!isa<VectorType>(I.getType()))
5395 Result = lowerRangeToAssertZExt(DAG, I, Result);
5396
5397 MaybeAlign Alignment = I.getRetAlign();
5398
5399 // Insert `assertalign` node if there's an alignment.
5400 if (InsertAssertAlign && Alignment) {
5401 Result =
5402 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5403 }
5404 }
5405
5406 setValue(&I, Result);
5407}
5408
5409/// GetSignificand - Get the significand and build it into a floating-point
5410/// number with exponent of 1:
5411///
5412/// Op = (Op & 0x007fffff) | 0x3f800000;
5413///
5414/// where Op is the hexadecimal representation of floating point value.
5416 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5417 DAG.getConstant(0x007fffff, dl, MVT::i32));
5418 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5419 DAG.getConstant(0x3f800000, dl, MVT::i32));
5420 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5421}
5422
5423/// GetExponent - Get the exponent:
5424///
5425/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5426///
5427/// where Op is the hexadecimal representation of floating point value.
5429 const TargetLowering &TLI, const SDLoc &dl) {
5430 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5431 DAG.getConstant(0x7f800000, dl, MVT::i32));
5432 SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
5433 DAG.getShiftAmountConstant(23, MVT::i32, dl));
5434 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5435 DAG.getConstant(127, dl, MVT::i32));
5436 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5437}
5438
5439/// getF32Constant - Get 32-bit floating point constant.
5441 const SDLoc &dl) {
5442 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5443 MVT::f32);
5444}
5445
5447 SelectionDAG &DAG) {
5448 // TODO: What fast-math-flags should be set on the floating-point nodes?
5449
5450 // IntegerPartOfX = ((int32_t)(t0);
5451 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5452
5453 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5454 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5455 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5456
5457 // IntegerPartOfX <<= 23;
5458 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5459 DAG.getShiftAmountConstant(23, MVT::i32, dl));
5460
5461 SDValue TwoToFractionalPartOfX;
5462 if (LimitFloatPrecision <= 6) {
5463 // For floating-point precision of 6:
5464 //
5465 // TwoToFractionalPartOfX =
5466 // 0.997535578f +
5467 // (0.735607626f + 0.252464424f * x) * x;
5468 //
5469 // error 0.0144103317, which is 6 bits
5470 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5471 getF32Constant(DAG, 0x3e814304, dl));
5472 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5473 getF32Constant(DAG, 0x3f3c50c8, dl));
5474 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5475 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5476 getF32Constant(DAG, 0x3f7f5e7e, dl));
5477 } else if (LimitFloatPrecision <= 12) {
5478 // For floating-point precision of 12:
5479 //
5480 // TwoToFractionalPartOfX =
5481 // 0.999892986f +
5482 // (0.696457318f +
5483 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5484 //
5485 // error 0.000107046256, which is 13 to 14 bits
5486 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5487 getF32Constant(DAG, 0x3da235e3, dl));
5488 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5489 getF32Constant(DAG, 0x3e65b8f3, dl));
5490 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5491 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5492 getF32Constant(DAG, 0x3f324b07, dl));
5493 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5494 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5495 getF32Constant(DAG, 0x3f7ff8fd, dl));
5496 } else { // LimitFloatPrecision <= 18
5497 // For floating-point precision of 18:
5498 //
5499 // TwoToFractionalPartOfX =
5500 // 0.999999982f +
5501 // (0.693148872f +
5502 // (0.240227044f +
5503 // (0.554906021e-1f +
5504 // (0.961591928e-2f +
5505 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5506 // error 2.47208000*10^(-7), which is better than 18 bits
5507 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5508 getF32Constant(DAG, 0x3924b03e, dl));
5509 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5510 getF32Constant(DAG, 0x3ab24b87, dl));
5511 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5512 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5513 getF32Constant(DAG, 0x3c1d8c17, dl));
5514 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5515 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5516 getF32Constant(DAG, 0x3d634a1d, dl));
5517 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5518 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5519 getF32Constant(DAG, 0x3e75fe14, dl));
5520 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5521 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5522 getF32Constant(DAG, 0x3f317234, dl));
5523 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5524 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5525 getF32Constant(DAG, 0x3f800000, dl));
5526 }
5527
5528 // Add the exponent into the result in integer domain.
5529 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5530 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5531 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5532}
5533
5534/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5535/// limited-precision mode.
5537 const TargetLowering &TLI, SDNodeFlags Flags) {
5538 if (Op.getValueType() == MVT::f32 &&
5540
5541 // Put the exponent in the right bit position for later addition to the
5542 // final result:
5543 //
5544 // t0 = Op * log2(e)
5545
5546 // TODO: What fast-math-flags should be set here?
5547 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5548 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5549 return getLimitedPrecisionExp2(t0, dl, DAG);
5550 }
5551
5552 // No special expansion.
5553 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5554}
5555
5556/// expandLog - Lower a log intrinsic. Handles the special sequences for
5557/// limited-precision mode.
5559 const TargetLowering &TLI, SDNodeFlags Flags) {
5560 // TODO: What fast-math-flags should be set on the floating-point nodes?
5561
5562 if (Op.getValueType() == MVT::f32 &&
5564 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5565
5566 // Scale the exponent by log(2).
5567 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5568 SDValue LogOfExponent =
5569 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5570 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5571
5572 // Get the significand and build it into a floating-point number with
5573 // exponent of 1.
5574 SDValue X = GetSignificand(DAG, Op1, dl);
5575
5576 SDValue LogOfMantissa;
5577 if (LimitFloatPrecision <= 6) {
5578 // For floating-point precision of 6:
5579 //
5580 // LogofMantissa =
5581 // -1.1609546f +
5582 // (1.4034025f - 0.23903021f * x) * x;
5583 //
5584 // error 0.0034276066, which is better than 8 bits
5585 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5586 getF32Constant(DAG, 0xbe74c456, dl));
5587 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5588 getF32Constant(DAG, 0x3fb3a2b1, dl));
5589 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5590 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5591 getF32Constant(DAG, 0x3f949a29, dl));
5592 } else if (LimitFloatPrecision <= 12) {
5593 // For floating-point precision of 12:
5594 //
5595 // LogOfMantissa =
5596 // -1.7417939f +
5597 // (2.8212026f +
5598 // (-1.4699568f +
5599 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5600 //
5601 // error 0.000061011436, which is 14 bits
5602 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5603 getF32Constant(DAG, 0xbd67b6d6, dl));
5604 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5605 getF32Constant(DAG, 0x3ee4f4b8, dl));
5606 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5607 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5608 getF32Constant(DAG, 0x3fbc278b, dl));
5609 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5610 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5611 getF32Constant(DAG, 0x40348e95, dl));
5612 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5613 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5614 getF32Constant(DAG, 0x3fdef31a, dl));
5615 } else { // LimitFloatPrecision <= 18
5616 // For floating-point precision of 18:
5617 //
5618 // LogOfMantissa =
5619 // -2.1072184f +
5620 // (4.2372794f +
5621 // (-3.7029485f +
5622 // (2.2781945f +
5623 // (-0.87823314f +
5624 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5625 //
5626 // error 0.0000023660568, which is better than 18 bits
5627 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5628 getF32Constant(DAG, 0xbc91e5ac, dl));
5629 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5630 getF32Constant(DAG, 0x3e4350aa, dl));
5631 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5632 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5633 getF32Constant(DAG, 0x3f60d3e3, dl));
5634 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5635 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5636 getF32Constant(DAG, 0x4011cdf0, dl));
5637 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5638 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5639 getF32Constant(DAG, 0x406cfd1c, dl));
5640 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5641 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5642 getF32Constant(DAG, 0x408797cb, dl));
5643 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5644 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5645 getF32Constant(DAG, 0x4006dcab, dl));
5646 }
5647
5648 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5649 }
5650
5651 // No special expansion.
5652 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5653}
5654
5655/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5656/// limited-precision mode.
5658 const TargetLowering &TLI, SDNodeFlags Flags) {
5659 // TODO: What fast-math-flags should be set on the floating-point nodes?
5660
5661 if (Op.getValueType() == MVT::f32 &&
5663 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5664
5665 // Get the exponent.
5666 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5667
5668 // Get the significand and build it into a floating-point number with
5669 // exponent of 1.
5670 SDValue X = GetSignificand(DAG, Op1, dl);
5671
5672 // Different possible minimax approximations of significand in
5673 // floating-point for various degrees of accuracy over [1,2].
5674 SDValue Log2ofMantissa;
5675 if (LimitFloatPrecision <= 6) {
5676 // For floating-point precision of 6:
5677 //
5678 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5679 //
5680 // error 0.0049451742, which is more than 7 bits
5681 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5682 getF32Constant(DAG, 0xbeb08fe0, dl));
5683 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5684 getF32Constant(DAG, 0x40019463, dl));
5685 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5686 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5687 getF32Constant(DAG, 0x3fd6633d, dl));
5688 } else if (LimitFloatPrecision <= 12) {
5689 // For floating-point precision of 12:
5690 //
5691 // Log2ofMantissa =
5692 // -2.51285454f +
5693 // (4.07009056f +
5694 // (-2.12067489f +
5695 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5696 //
5697 // error 0.0000876136000, which is better than 13 bits
5698 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5699 getF32Constant(DAG, 0xbda7262e, dl));
5700 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5701 getF32Constant(DAG, 0x3f25280b, dl));
5702 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5703 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5704 getF32Constant(DAG, 0x4007b923, dl));
5705 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5706 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5707 getF32Constant(DAG, 0x40823e2f, dl));
5708 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5709 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5710 getF32Constant(DAG, 0x4020d29c, dl));
5711 } else { // LimitFloatPrecision <= 18
5712 // For floating-point precision of 18:
5713 //
5714 // Log2ofMantissa =
5715 // -3.0400495f +
5716 // (6.1129976f +
5717 // (-5.3420409f +
5718 // (3.2865683f +
5719 // (-1.2669343f +
5720 // (0.27515199f -
5721 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5722 //
5723 // error 0.0000018516, which is better than 18 bits
5724 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5725 getF32Constant(DAG, 0xbcd2769e, dl));
5726 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5727 getF32Constant(DAG, 0x3e8ce0b9, dl));
5728 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5729 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5730 getF32Constant(DAG, 0x3fa22ae7, dl));
5731 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5732 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5733 getF32Constant(DAG, 0x40525723, dl));
5734 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5735 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5736 getF32Constant(DAG, 0x40aaf200, dl));
5737 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5738 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5739 getF32Constant(DAG, 0x40c39dad, dl));
5740 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5741 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5742 getF32Constant(DAG, 0x4042902c, dl));
5743 }
5744
5745 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5746 }
5747
5748 // No special expansion.
5749 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5750}
5751
5752/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5753/// limited-precision mode.
5755 const TargetLowering &TLI, SDNodeFlags Flags) {
5756 // TODO: What fast-math-flags should be set on the floating-point nodes?
5757
5758 if (Op.getValueType() == MVT::f32 &&
5760 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5761
5762 // Scale the exponent by log10(2) [0.30102999f].
5763 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5764 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5765 getF32Constant(DAG, 0x3e9a209a, dl));
5766
5767 // Get the significand and build it into a floating-point number with
5768 // exponent of 1.
5769 SDValue X = GetSignificand(DAG, Op1, dl);
5770
5771 SDValue Log10ofMantissa;
5772 if (LimitFloatPrecision <= 6) {
5773 // For floating-point precision of 6:
5774 //
5775 // Log10ofMantissa =
5776 // -0.50419619f +
5777 // (0.60948995f - 0.10380950f * x) * x;
5778 //
5779 // error 0.0014886165, which is 6 bits
5780 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5781 getF32Constant(DAG, 0xbdd49a13, dl));
5782 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5783 getF32Constant(DAG, 0x3f1c0789, dl));
5784 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5785 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5786 getF32Constant(DAG, 0x3f011300, dl));
5787 } else if (LimitFloatPrecision <= 12) {
5788 // For floating-point precision of 12:
5789 //
5790 // Log10ofMantissa =
5791 // -0.64831180f +
5792 // (0.91751397f +
5793 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5794 //
5795 // error 0.00019228036, which is better than 12 bits
5796 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5797 getF32Constant(DAG, 0x3d431f31, dl));
5798 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5799 getF32Constant(DAG, 0x3ea21fb2, dl));
5800 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5801 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5802 getF32Constant(DAG, 0x3f6ae232, dl));
5803 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5804 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5805 getF32Constant(DAG, 0x3f25f7c3, dl));
5806 } else { // LimitFloatPrecision <= 18
5807 // For floating-point precision of 18:
5808 //
5809 // Log10ofMantissa =
5810 // -0.84299375f +
5811 // (1.5327582f +
5812 // (-1.0688956f +
5813 // (0.49102474f +
5814 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5815 //
5816 // error 0.0000037995730, which is better than 18 bits
5817 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5818 getF32Constant(DAG, 0x3c5d51ce, dl));
5819 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5820 getF32Constant(DAG, 0x3e00685a, dl));
5821 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5822 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5823 getF32Constant(DAG, 0x3efb6798, dl));
5824 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5825 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5826 getF32Constant(DAG, 0x3f88d192, dl));
5827 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5828 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5829 getF32Constant(DAG, 0x3fc4316c, dl));
5830 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5831 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5832 getF32Constant(DAG, 0x3f57ce70, dl));
5833 }
5834
5835 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5836 }
5837
5838 // No special expansion.
5839 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5840}
5841
5842/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5843/// limited-precision mode.
5845 const TargetLowering &TLI, SDNodeFlags Flags) {
5846 if (Op.getValueType() == MVT::f32 &&
5848 return getLimitedPrecisionExp2(Op, dl, DAG);
5849
5850 // No special expansion.
5851 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5852}
5853
5854/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5855/// limited-precision mode with x == 10.0f.
5857 SelectionDAG &DAG, const TargetLowering &TLI,
5858 SDNodeFlags Flags) {
5859 bool IsExp10 = false;
5860 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5863 APFloat Ten(10.0f);
5864 IsExp10 = LHSC->isExactlyValue(Ten);
5865 }
5866 }
5867
5868 // TODO: What fast-math-flags should be set on the FMUL node?
5869 if (IsExp10) {
5870 // Put the exponent in the right bit position for later addition to the
5871 // final result:
5872 //
5873 // #define LOG2OF10 3.3219281f
5874 // t0 = Op * LOG2OF10;
5875 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5876 getF32Constant(DAG, 0x40549a78, dl));
5877 return getLimitedPrecisionExp2(t0, dl, DAG);
5878 }
5879
5880 // No special expansion.
5881 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5882}
5883
5884/// ExpandPowI - Expand a llvm.powi intrinsic.
5886 SelectionDAG &DAG) {
5887 // If RHS is a constant, we can expand this out to a multiplication tree if
5888 // it's beneficial on the target, otherwise we end up lowering to a call to
5889 // __powidf2 (for example).
5891 unsigned Val = RHSC->getSExtValue();
5892
5893 // powi(x, 0) -> 1.0
5894 if (Val == 0)
5895 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5896
5898 Val, DAG.shouldOptForSize())) {
5899 // Get the exponent as a positive value.
5900 if ((int)Val < 0)
5901 Val = -Val;
5902 // We use the simple binary decomposition method to generate the multiply
5903 // sequence. There are more optimal ways to do this (for example,
5904 // powi(x,15) generates one more multiply than it should), but this has
5905 // the benefit of being both really simple and much better than a libcall.
5906 SDValue Res; // Logically starts equal to 1.0
5907 SDValue CurSquare = LHS;
5908 // TODO: Intrinsics should have fast-math-flags that propagate to these
5909 // nodes.
5910 while (Val) {
5911 if (Val & 1) {
5912 if (Res.getNode())
5913 Res =
5914 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5915 else
5916 Res = CurSquare; // 1.0*CurSquare.
5917 }
5918
5919 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5920 CurSquare, CurSquare);
5921 Val >>= 1;
5922 }
5923
5924 // If the original was negative, invert the result, producing 1/(x*x*x).
5925 if (RHSC->getSExtValue() < 0)
5926 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5927 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5928 return Res;
5929 }
5930 }
5931
5932 // Otherwise, expand to a libcall.
5933 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5934}
5935
5936static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5937 SDValue LHS, SDValue RHS, SDValue Scale,
5938 SelectionDAG &DAG, const TargetLowering &TLI) {
5939 EVT VT = LHS.getValueType();
5940 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5941 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5942 LLVMContext &Ctx = *DAG.getContext();
5943
5944 // If the type is legal but the operation isn't, this node might survive all
5945 // the way to operation legalization. If we end up there and we do not have
5946 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5947 // node.
5948
5949 // Coax the legalizer into expanding the node during type legalization instead
5950 // by bumping the size by one bit. This will force it to Promote, enabling the
5951 // early expansion and avoiding the need to expand later.
5952
5953 // We don't have to do this if Scale is 0; that can always be expanded, unless
5954 // it's a saturating signed operation. Those can experience true integer
5955 // division overflow, a case which we must avoid.
5956
5957 // FIXME: We wouldn't have to do this (or any of the early
5958 // expansion/promotion) if it was possible to expand a libcall of an
5959 // illegal type during operation legalization. But it's not, so things
5960 // get a bit hacky.
5961 unsigned ScaleInt = Scale->getAsZExtVal();
5962 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5963 (TLI.isTypeLegal(VT) ||
5964 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5966 Opcode, VT, ScaleInt);
5967 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5968 EVT PromVT;
5969 if (VT.isScalarInteger())
5970 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5971 else if (VT.isVector()) {
5972 PromVT = VT.getVectorElementType();
5973 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5974 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5975 } else
5976 llvm_unreachable("Wrong VT for DIVFIX?");
5977 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
5978 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
5979 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5980 // For saturating operations, we need to shift up the LHS to get the
5981 // proper saturation width, and then shift down again afterwards.
5982 if (Saturating)
5983 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5984 DAG.getConstant(1, DL, ShiftTy));
5985 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5986 if (Saturating)
5987 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5988 DAG.getConstant(1, DL, ShiftTy));
5989 return DAG.getZExtOrTrunc(Res, DL, VT);
5990 }
5991 }
5992
5993 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5994}
5995
5996// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5997// bitcasted, or split argument. Returns a list of <Register, size in bits>
5998static void
5999getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
6000 const SDValue &N) {
6001 switch (N.getOpcode()) {
6002 case ISD::CopyFromReg: {
6003 SDValue Op = N.getOperand(1);
6004 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
6005 Op.getValueType().getSizeInBits());
6006 return;
6007 }
6008 case ISD::BITCAST:
6009 case ISD::AssertZext:
6010 case ISD::AssertSext:
6011 case ISD::TRUNCATE:
6012 getUnderlyingArgRegs(Regs, N.getOperand(0));
6013 return;
6014 case ISD::BUILD_PAIR:
6015 case ISD::BUILD_VECTOR:
6017 for (SDValue Op : N->op_values())
6018 getUnderlyingArgRegs(Regs, Op);
6019 return;
6020 default:
6021 return;
6022 }
6023}
6024
6025/// If the DbgValueInst is a dbg_value of a function argument, create the
6026/// corresponding DBG_VALUE machine instruction for it now. At the end of
6027/// instruction selection, they will be inserted to the entry BB.
6028/// We don't currently support this for variadic dbg_values, as they shouldn't
6029/// appear for function arguments or in the prologue.
6030bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6031 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6032 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6033 const Argument *Arg = dyn_cast<Argument>(V);
6034 if (!Arg)
6035 return false;
6036
6037 MachineFunction &MF = DAG.getMachineFunction();
6038 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6039
6040 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6041 // we've been asked to pursue.
6042 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6043 bool Indirect) {
6044 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6045 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6046 // pointing at the VReg, which will be patched up later.
6047 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
6049 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6050 /* isKill */ false, /* isDead */ false,
6051 /* isUndef */ false, /* isEarlyClobber */ false,
6052 /* SubReg */ 0, /* isDebug */ true)});
6053
6054 auto *NewDIExpr = FragExpr;
6055 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6056 // the DIExpression.
6057 if (Indirect)
6058 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
6060 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
6061 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
6062 } else {
6063 // Create a completely standard DBG_VALUE.
6064 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
6065 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
6066 }
6067 };
6068
6069 if (Kind == FuncArgumentDbgValueKind::Value) {
6070 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6071 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6072 // the entry block.
6073 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6074 if (!IsInEntryBlock)
6075 return false;
6076
6077 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6078 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6079 // variable that also is a param.
6080 //
6081 // Although, if we are at the top of the entry block already, we can still
6082 // emit using ArgDbgValue. This might catch some situations when the
6083 // dbg.value refers to an argument that isn't used in the entry block, so
6084 // any CopyToReg node would be optimized out and the only way to express
6085 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6086 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6087 // we should only emit as ArgDbgValue if the Variable is an argument to the
6088 // current function, and the dbg.value intrinsic is found in the entry
6089 // block.
6090 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6091 !DL->getInlinedAt();
6092 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6093 if (!IsInPrologue && !VariableIsFunctionInputArg)
6094 return false;
6095
6096 // Here we assume that a function argument on IR level only can be used to
6097 // describe one input parameter on source level. If we for example have
6098 // source code like this
6099 //
6100 // struct A { long x, y; };
6101 // void foo(struct A a, long b) {
6102 // ...
6103 // b = a.x;
6104 // ...
6105 // }
6106 //
6107 // and IR like this
6108 //
6109 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6110 // entry:
6111 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6112 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6113 // call void @llvm.dbg.value(metadata i32 %b, "b",
6114 // ...
6115 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6116 // ...
6117 //
6118 // then the last dbg.value is describing a parameter "b" using a value that
6119 // is an argument. But since we already has used %a1 to describe a parameter
6120 // we should not handle that last dbg.value here (that would result in an
6121 // incorrect hoisting of the DBG_VALUE to the function entry).
6122 // Notice that we allow one dbg.value per IR level argument, to accommodate
6123 // for the situation with fragments above.
6124 // If there is no node for the value being handled, we return true to skip
6125 // the normal generation of debug info, as it would kill existing debug
6126 // info for the parameter in case of duplicates.
6127 if (VariableIsFunctionInputArg) {
6128 unsigned ArgNo = Arg->getArgNo();
6129 if (ArgNo >= FuncInfo.DescribedArgs.size())
6130 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6131 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6132 return !NodeMap[V].getNode();
6133 FuncInfo.DescribedArgs.set(ArgNo);
6134 }
6135 }
6136
6137 bool IsIndirect = false;
6138 std::optional<MachineOperand> Op;
6139 // Some arguments' frame index is recorded during argument lowering.
6140 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6141 if (FI != std::numeric_limits<int>::max())
6143
6145 if (!Op && N.getNode()) {
6146 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6147 Register Reg;
6148 if (ArgRegsAndSizes.size() == 1)
6149 Reg = ArgRegsAndSizes.front().first;
6150
6151 if (Reg && Reg.isVirtual()) {
6152 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6153 Register PR = RegInfo.getLiveInPhysReg(Reg);
6154 if (PR)
6155 Reg = PR;
6156 }
6157 if (Reg) {
6159 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6160 }
6161 }
6162
6163 if (!Op && N.getNode()) {
6164 // Check if frame index is available.
6165 SDValue LCandidate = peekThroughBitcasts(N);
6166 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6167 if (FrameIndexSDNode *FINode =
6168 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6169 Op = MachineOperand::CreateFI(FINode->getIndex());
6170 }
6171
6172 if (!Op) {
6173 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6174 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<Register, TypeSize>>
6175 SplitRegs) {
6176 unsigned Offset = 0;
6177 for (const auto &RegAndSize : SplitRegs) {
6178 // If the expression is already a fragment, the current register
6179 // offset+size might extend beyond the fragment. In this case, only
6180 // the register bits that are inside the fragment are relevant.
6181 int RegFragmentSizeInBits = RegAndSize.second;
6182 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6183 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6184 // The register is entirely outside the expression fragment,
6185 // so is irrelevant for debug info.
6186 if (Offset >= ExprFragmentSizeInBits)
6187 break;
6188 // The register is partially outside the expression fragment, only
6189 // the low bits within the fragment are relevant for debug info.
6190 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6191 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6192 }
6193 }
6194
6195 auto FragmentExpr = DIExpression::createFragmentExpression(
6196 Expr, Offset, RegFragmentSizeInBits);
6197 Offset += RegAndSize.second;
6198 // If a valid fragment expression cannot be created, the variable's
6199 // correct value cannot be determined and so it is set as poison.
6200 if (!FragmentExpr) {
6201 SDDbgValue *SDV = DAG.getConstantDbgValue(
6202 Variable, Expr, PoisonValue::get(V->getType()), DL, SDNodeOrder);
6203 DAG.AddDbgValue(SDV, false);
6204 continue;
6205 }
6206 MachineInstr *NewMI =
6207 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6208 Kind != FuncArgumentDbgValueKind::Value);
6209 FuncInfo.ArgDbgValues.push_back(NewMI);
6210 }
6211 };
6212
6213 // Check if ValueMap has reg number.
6215 VMI = FuncInfo.ValueMap.find(V);
6216 if (VMI != FuncInfo.ValueMap.end()) {
6217 const auto &TLI = DAG.getTargetLoweringInfo();
6218 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6219 V->getType(), std::nullopt);
6220 if (RFV.occupiesMultipleRegs()) {
6221 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6222 return true;
6223 }
6224
6225 Op = MachineOperand::CreateReg(VMI->second, false);
6226 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6227 } else if (ArgRegsAndSizes.size() > 1) {
6228 // This was split due to the calling convention, and no virtual register
6229 // mapping exists for the value.
6230 splitMultiRegDbgValue(ArgRegsAndSizes);
6231 return true;
6232 }
6233 }
6234
6235 if (!Op)
6236 return false;
6237
6238 assert(Variable->isValidLocationForIntrinsic(DL) &&
6239 "Expected inlined-at fields to agree");
6240 MachineInstr *NewMI = nullptr;
6241
6242 if (Op->isReg())
6243 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6244 else
6245 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6246 Variable, Expr);
6247
6248 // Otherwise, use ArgDbgValues.
6249 FuncInfo.ArgDbgValues.push_back(NewMI);
6250 return true;
6251}
6252
6253/// Return the appropriate SDDbgValue based on N.
6254SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6255 DILocalVariable *Variable,
6256 DIExpression *Expr,
6257 const DebugLoc &dl,
6258 unsigned DbgSDNodeOrder) {
6259 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6260 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6261 // stack slot locations.
6262 //
6263 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6264 // debug values here after optimization:
6265 //
6266 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6267 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6268 //
6269 // Both describe the direct values of their associated variables.
6270 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6271 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6272 }
6273 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6274 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6275}
6276
6277static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6278 switch (Intrinsic) {
6279 case Intrinsic::smul_fix:
6280 return ISD::SMULFIX;
6281 case Intrinsic::umul_fix:
6282 return ISD::UMULFIX;
6283 case Intrinsic::smul_fix_sat:
6284 return ISD::SMULFIXSAT;
6285 case Intrinsic::umul_fix_sat:
6286 return ISD::UMULFIXSAT;
6287 case Intrinsic::sdiv_fix:
6288 return ISD::SDIVFIX;
6289 case Intrinsic::udiv_fix:
6290 return ISD::UDIVFIX;
6291 case Intrinsic::sdiv_fix_sat:
6292 return ISD::SDIVFIXSAT;
6293 case Intrinsic::udiv_fix_sat:
6294 return ISD::UDIVFIXSAT;
6295 default:
6296 llvm_unreachable("Unhandled fixed point intrinsic");
6297 }
6298}
6299
6300/// Given a @llvm.call.preallocated.setup, return the corresponding
6301/// preallocated call.
6302static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6303 assert(cast<CallBase>(PreallocatedSetup)
6305 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6306 "expected call_preallocated_setup Value");
6307 for (const auto *U : PreallocatedSetup->users()) {
6308 auto *UseCall = cast<CallBase>(U);
6309 const Function *Fn = UseCall->getCalledFunction();
6310 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6311 return UseCall;
6312 }
6313 }
6314 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6315}
6316
6317/// If DI is a debug value with an EntryValue expression, lower it using the
6318/// corresponding physical register of the associated Argument value
6319/// (guaranteed to exist by the verifier).
6320bool SelectionDAGBuilder::visitEntryValueDbgValue(
6321 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6322 DIExpression *Expr, DebugLoc DbgLoc) {
6323 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6324 return false;
6325
6326 // These properties are guaranteed by the verifier.
6327 const Argument *Arg = cast<Argument>(Values[0]);
6328 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6329
6330 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6331 if (ArgIt == FuncInfo.ValueMap.end()) {
6332 LLVM_DEBUG(
6333 dbgs() << "Dropping dbg.value: expression is entry_value but "
6334 "couldn't find an associated register for the Argument\n");
6335 return true;
6336 }
6337 Register ArgVReg = ArgIt->getSecond();
6338
6339 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6340 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6341 SDDbgValue *SDV = DAG.getVRegDbgValue(
6342 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6343 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6344 return true;
6345 }
6346 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6347 "couldn't find a physical register\n");
6348 return true;
6349}
6350
6351/// Lower the call to the specified intrinsic function.
6352void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6353 unsigned Intrinsic) {
6354 SDLoc sdl = getCurSDLoc();
6355 switch (Intrinsic) {
6356 case Intrinsic::experimental_convergence_anchor:
6357 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6358 break;
6359 case Intrinsic::experimental_convergence_entry:
6360 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6361 break;
6362 case Intrinsic::experimental_convergence_loop: {
6363 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6364 auto *Token = Bundle->Inputs[0].get();
6365 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6366 getValue(Token)));
6367 break;
6368 }
6369 }
6370}
6371
6372void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6373 unsigned IntrinsicID) {
6374 // For now, we're only lowering an 'add' histogram.
6375 // We can add others later, e.g. saturating adds, min/max.
6376 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6377 "Tried to lower unsupported histogram type");
6378 SDLoc sdl = getCurSDLoc();
6379 Value *Ptr = I.getOperand(0);
6380 SDValue Inc = getValue(I.getOperand(1));
6381 SDValue Mask = getValue(I.getOperand(2));
6382
6383 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6384 DataLayout TargetDL = DAG.getDataLayout();
6385 EVT VT = Inc.getValueType();
6386 Align Alignment = DAG.getEVTAlign(VT);
6387
6388 const MDNode *Ranges = getRangeMetadata(I);
6389
6390 SDValue Root = DAG.getRoot();
6391 SDValue Base;
6392 SDValue Index;
6393 SDValue Scale;
6394 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
6395 I.getParent(), VT.getScalarStoreSize());
6396
6397 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6398
6399 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6400 MachinePointerInfo(AS),
6402 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
6403
6404 if (!UniformBase) {
6405 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
6406 Index = getValue(Ptr);
6407 Scale =
6408 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
6409 }
6410
6411 EVT IdxVT = Index.getValueType();
6412 EVT EltTy = IdxVT.getVectorElementType();
6413 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
6414 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
6415 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
6416 }
6417
6418 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6419
6420 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6421 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
6422 Ops, MMO, ISD::SIGNED_SCALED);
6423
6424 setValue(&I, Histogram);
6425 DAG.setRoot(Histogram);
6426}
6427
6428void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6429 unsigned Intrinsic) {
6430 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6431 "Tried lowering invalid vector extract last");
6432 SDLoc sdl = getCurSDLoc();
6433 const DataLayout &Layout = DAG.getDataLayout();
6434 SDValue Data = getValue(I.getOperand(0));
6435 SDValue Mask = getValue(I.getOperand(1));
6436
6437 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6438 EVT ResVT = TLI.getValueType(Layout, I.getType());
6439
6440 EVT ExtVT = TLI.getVectorIdxTy(Layout);
6441 SDValue Idx = DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, sdl, ExtVT, Mask);
6442 SDValue Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl, ResVT, Data, Idx);
6443
6444 Value *Default = I.getOperand(2);
6446 SDValue PassThru = getValue(Default);
6447 EVT BoolVT = Mask.getValueType().getScalarType();
6448 SDValue AnyActive = DAG.getNode(ISD::VECREDUCE_OR, sdl, BoolVT, Mask);
6449 Result = DAG.getSelect(sdl, ResVT, AnyActive, Result, PassThru);
6450 }
6451
6452 setValue(&I, Result);
6453}
6454
6455/// Lower the call to the specified intrinsic function.
6456void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6457 unsigned Intrinsic) {
6458 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6459 SDLoc sdl = getCurSDLoc();
6460 DebugLoc dl = getCurDebugLoc();
6461 SDValue Res;
6462
6463 SDNodeFlags Flags;
6464 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6465 Flags.copyFMF(*FPOp);
6466
6467 switch (Intrinsic) {
6468 default:
6469 // By default, turn this into a target intrinsic node.
6470 visitTargetIntrinsic(I, Intrinsic);
6471 return;
6472 case Intrinsic::vscale: {
6473 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6474 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6475 return;
6476 }
6477 case Intrinsic::vastart: visitVAStart(I); return;
6478 case Intrinsic::vaend: visitVAEnd(I); return;
6479 case Intrinsic::vacopy: visitVACopy(I); return;
6480 case Intrinsic::returnaddress:
6481 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
6482 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6483 getValue(I.getArgOperand(0))));
6484 return;
6485 case Intrinsic::addressofreturnaddress:
6486 setValue(&I,
6487 DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
6488 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6489 return;
6490 case Intrinsic::sponentry:
6491 setValue(&I,
6492 DAG.getNode(ISD::SPONENTRY, sdl,
6493 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6494 return;
6495 case Intrinsic::frameaddress:
6496 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
6497 TLI.getFrameIndexTy(DAG.getDataLayout()),
6498 getValue(I.getArgOperand(0))));
6499 return;
6500 case Intrinsic::read_volatile_register:
6501 case Intrinsic::read_register: {
6502 Value *Reg = I.getArgOperand(0);
6503 SDValue Chain = getRoot();
6505 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6506 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6507 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6508 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6509 setValue(&I, Res);
6510 DAG.setRoot(Res.getValue(1));
6511 return;
6512 }
6513 case Intrinsic::write_register: {
6514 Value *Reg = I.getArgOperand(0);
6515 Value *RegValue = I.getArgOperand(1);
6516 SDValue Chain = getRoot();
6518 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6519 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6520 RegName, getValue(RegValue)));
6521 return;
6522 }
6523 case Intrinsic::memcpy:
6524 case Intrinsic::memcpy_inline: {
6525 const auto &MCI = cast<MemCpyInst>(I);
6526 SDValue Dst = getValue(I.getArgOperand(0));
6527 SDValue Src = getValue(I.getArgOperand(1));
6528 SDValue Size = getValue(I.getArgOperand(2));
6529 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6530 "memcpy_inline needs constant size");
6531 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6532 Align DstAlign = MCI.getDestAlign().valueOrOne();
6533 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6534 Align Alignment = std::min(DstAlign, SrcAlign);
6535 bool isVol = MCI.isVolatile();
6536 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6537 // node.
6538 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6539 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol,
6540 MCI.isForceInlined(), &I, std::nullopt,
6541 MachinePointerInfo(I.getArgOperand(0)),
6542 MachinePointerInfo(I.getArgOperand(1)),
6543 I.getAAMetadata(), BatchAA);
6544 updateDAGForMaybeTailCall(MC);
6545 return;
6546 }
6547 case Intrinsic::memset:
6548 case Intrinsic::memset_inline: {
6549 const auto &MSII = cast<MemSetInst>(I);
6550 SDValue Dst = getValue(I.getArgOperand(0));
6551 SDValue Value = getValue(I.getArgOperand(1));
6552 SDValue Size = getValue(I.getArgOperand(2));
6553 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6554 "memset_inline needs constant size");
6555 // @llvm.memset defines 0 and 1 to both mean no alignment.
6556 Align DstAlign = MSII.getDestAlign().valueOrOne();
6557 bool isVol = MSII.isVolatile();
6558 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6559 SDValue MC = DAG.getMemset(
6560 Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(),
6561 &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6562 updateDAGForMaybeTailCall(MC);
6563 return;
6564 }
6565 case Intrinsic::memmove: {
6566 const auto &MMI = cast<MemMoveInst>(I);
6567 SDValue Op1 = getValue(I.getArgOperand(0));
6568 SDValue Op2 = getValue(I.getArgOperand(1));
6569 SDValue Op3 = getValue(I.getArgOperand(2));
6570 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6571 Align DstAlign = MMI.getDestAlign().valueOrOne();
6572 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6573 Align Alignment = std::min(DstAlign, SrcAlign);
6574 bool isVol = MMI.isVolatile();
6575 // FIXME: Support passing different dest/src alignments to the memmove DAG
6576 // node.
6577 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6578 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, &I,
6579 /* OverrideTailCall */ std::nullopt,
6580 MachinePointerInfo(I.getArgOperand(0)),
6581 MachinePointerInfo(I.getArgOperand(1)),
6582 I.getAAMetadata(), BatchAA);
6583 updateDAGForMaybeTailCall(MM);
6584 return;
6585 }
6586 case Intrinsic::memcpy_element_unordered_atomic: {
6587 auto &MI = cast<AnyMemCpyInst>(I);
6588 SDValue Dst = getValue(MI.getRawDest());
6589 SDValue Src = getValue(MI.getRawSource());
6590 SDValue Length = getValue(MI.getLength());
6591
6592 Type *LengthTy = MI.getLength()->getType();
6593 unsigned ElemSz = MI.getElementSizeInBytes();
6594 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6595 SDValue MC =
6596 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6597 isTC, MachinePointerInfo(MI.getRawDest()),
6598 MachinePointerInfo(MI.getRawSource()));
6599 updateDAGForMaybeTailCall(MC);
6600 return;
6601 }
6602 case Intrinsic::memmove_element_unordered_atomic: {
6603 auto &MI = cast<AnyMemMoveInst>(I);
6604 SDValue Dst = getValue(MI.getRawDest());
6605 SDValue Src = getValue(MI.getRawSource());
6606 SDValue Length = getValue(MI.getLength());
6607
6608 Type *LengthTy = MI.getLength()->getType();
6609 unsigned ElemSz = MI.getElementSizeInBytes();
6610 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6611 SDValue MC =
6612 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6613 isTC, MachinePointerInfo(MI.getRawDest()),
6614 MachinePointerInfo(MI.getRawSource()));
6615 updateDAGForMaybeTailCall(MC);
6616 return;
6617 }
6618 case Intrinsic::memset_element_unordered_atomic: {
6619 auto &MI = cast<AnyMemSetInst>(I);
6620 SDValue Dst = getValue(MI.getRawDest());
6621 SDValue Val = getValue(MI.getValue());
6622 SDValue Length = getValue(MI.getLength());
6623
6624 Type *LengthTy = MI.getLength()->getType();
6625 unsigned ElemSz = MI.getElementSizeInBytes();
6626 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6627 SDValue MC =
6628 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
6629 isTC, MachinePointerInfo(MI.getRawDest()));
6630 updateDAGForMaybeTailCall(MC);
6631 return;
6632 }
6633 case Intrinsic::call_preallocated_setup: {
6634 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6635 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6636 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6637 getRoot(), SrcValue);
6638 setValue(&I, Res);
6639 DAG.setRoot(Res);
6640 return;
6641 }
6642 case Intrinsic::call_preallocated_arg: {
6643 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6644 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6645 SDValue Ops[3];
6646 Ops[0] = getRoot();
6647 Ops[1] = SrcValue;
6648 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6649 MVT::i32); // arg index
6650 SDValue Res = DAG.getNode(
6651 ISD::PREALLOCATED_ARG, sdl,
6652 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6653 setValue(&I, Res);
6654 DAG.setRoot(Res.getValue(1));
6655 return;
6656 }
6657
6658 case Intrinsic::eh_typeid_for: {
6659 // Find the type id for the given typeinfo.
6660 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6661 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6662 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6663 setValue(&I, Res);
6664 return;
6665 }
6666
6667 case Intrinsic::eh_return_i32:
6668 case Intrinsic::eh_return_i64:
6669 DAG.getMachineFunction().setCallsEHReturn(true);
6670 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
6671 MVT::Other,
6673 getValue(I.getArgOperand(0)),
6674 getValue(I.getArgOperand(1))));
6675 return;
6676 case Intrinsic::eh_unwind_init:
6677 DAG.getMachineFunction().setCallsUnwindInit(true);
6678 return;
6679 case Intrinsic::eh_dwarf_cfa:
6680 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
6681 TLI.getPointerTy(DAG.getDataLayout()),
6682 getValue(I.getArgOperand(0))));
6683 return;
6684 case Intrinsic::eh_sjlj_callsite: {
6685 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6686 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
6687
6688 FuncInfo.setCurrentCallSite(CI->getZExtValue());
6689 return;
6690 }
6691 case Intrinsic::eh_sjlj_functioncontext: {
6692 // Get and store the index of the function context.
6693 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6694 AllocaInst *FnCtx =
6695 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6696 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6698 return;
6699 }
6700 case Intrinsic::eh_sjlj_setjmp: {
6701 SDValue Ops[2];
6702 Ops[0] = getRoot();
6703 Ops[1] = getValue(I.getArgOperand(0));
6704 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
6705 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6706 setValue(&I, Op.getValue(0));
6707 DAG.setRoot(Op.getValue(1));
6708 return;
6709 }
6710 case Intrinsic::eh_sjlj_longjmp:
6711 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6712 getRoot(), getValue(I.getArgOperand(0))));
6713 return;
6714 case Intrinsic::eh_sjlj_setup_dispatch:
6715 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
6716 getRoot()));
6717 return;
6718 case Intrinsic::masked_gather:
6719 visitMaskedGather(I);
6720 return;
6721 case Intrinsic::masked_load:
6722 visitMaskedLoad(I);
6723 return;
6724 case Intrinsic::masked_scatter:
6725 visitMaskedScatter(I);
6726 return;
6727 case Intrinsic::masked_store:
6728 visitMaskedStore(I);
6729 return;
6730 case Intrinsic::masked_expandload:
6731 visitMaskedLoad(I, true /* IsExpanding */);
6732 return;
6733 case Intrinsic::masked_compressstore:
6734 visitMaskedStore(I, true /* IsCompressing */);
6735 return;
6736 case Intrinsic::powi:
6737 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6738 getValue(I.getArgOperand(1)), DAG));
6739 return;
6740 case Intrinsic::log:
6741 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6742 return;
6743 case Intrinsic::log2:
6744 setValue(&I,
6745 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6746 return;
6747 case Intrinsic::log10:
6748 setValue(&I,
6749 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6750 return;
6751 case Intrinsic::exp:
6752 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6753 return;
6754 case Intrinsic::exp2:
6755 setValue(&I,
6756 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6757 return;
6758 case Intrinsic::pow:
6759 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6760 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6761 return;
6762 case Intrinsic::sqrt:
6763 case Intrinsic::fabs:
6764 case Intrinsic::sin:
6765 case Intrinsic::cos:
6766 case Intrinsic::tan:
6767 case Intrinsic::asin:
6768 case Intrinsic::acos:
6769 case Intrinsic::atan:
6770 case Intrinsic::sinh:
6771 case Intrinsic::cosh:
6772 case Intrinsic::tanh:
6773 case Intrinsic::exp10:
6774 case Intrinsic::floor:
6775 case Intrinsic::ceil:
6776 case Intrinsic::trunc:
6777 case Intrinsic::rint:
6778 case Intrinsic::nearbyint:
6779 case Intrinsic::round:
6780 case Intrinsic::roundeven:
6781 case Intrinsic::canonicalize: {
6782 unsigned Opcode;
6783 // clang-format off
6784 switch (Intrinsic) {
6785 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6786 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6787 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6788 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6789 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6790 case Intrinsic::tan: Opcode = ISD::FTAN; break;
6791 case Intrinsic::asin: Opcode = ISD::FASIN; break;
6792 case Intrinsic::acos: Opcode = ISD::FACOS; break;
6793 case Intrinsic::atan: Opcode = ISD::FATAN; break;
6794 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
6795 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
6796 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
6797 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6798 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6799 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6800 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6801 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6802 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6803 case Intrinsic::round: Opcode = ISD::FROUND; break;
6804 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6805 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6806 }
6807 // clang-format on
6808
6809 setValue(&I, DAG.getNode(Opcode, sdl,
6810 getValue(I.getArgOperand(0)).getValueType(),
6811 getValue(I.getArgOperand(0)), Flags));
6812 return;
6813 }
6814 case Intrinsic::atan2:
6815 setValue(&I, DAG.getNode(ISD::FATAN2, sdl,
6816 getValue(I.getArgOperand(0)).getValueType(),
6817 getValue(I.getArgOperand(0)),
6818 getValue(I.getArgOperand(1)), Flags));
6819 return;
6820 case Intrinsic::lround:
6821 case Intrinsic::llround:
6822 case Intrinsic::lrint:
6823 case Intrinsic::llrint: {
6824 unsigned Opcode;
6825 // clang-format off
6826 switch (Intrinsic) {
6827 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6828 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6829 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6830 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6831 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6832 }
6833 // clang-format on
6834
6835 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6836 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6837 getValue(I.getArgOperand(0))));
6838 return;
6839 }
6840 case Intrinsic::minnum:
6841 setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
6842 getValue(I.getArgOperand(0)).getValueType(),
6843 getValue(I.getArgOperand(0)),
6844 getValue(I.getArgOperand(1)), Flags));
6845 return;
6846 case Intrinsic::maxnum:
6847 setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
6848 getValue(I.getArgOperand(0)).getValueType(),
6849 getValue(I.getArgOperand(0)),
6850 getValue(I.getArgOperand(1)), Flags));
6851 return;
6852 case Intrinsic::minimum:
6853 setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
6854 getValue(I.getArgOperand(0)).getValueType(),
6855 getValue(I.getArgOperand(0)),
6856 getValue(I.getArgOperand(1)), Flags));
6857 return;
6858 case Intrinsic::maximum:
6859 setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
6860 getValue(I.getArgOperand(0)).getValueType(),
6861 getValue(I.getArgOperand(0)),
6862 getValue(I.getArgOperand(1)), Flags));
6863 return;
6864 case Intrinsic::minimumnum:
6865 setValue(&I, DAG.getNode(ISD::FMINIMUMNUM, sdl,
6866 getValue(I.getArgOperand(0)).getValueType(),
6867 getValue(I.getArgOperand(0)),
6868 getValue(I.getArgOperand(1)), Flags));
6869 return;
6870 case Intrinsic::maximumnum:
6871 setValue(&I, DAG.getNode(ISD::FMAXIMUMNUM, sdl,
6872 getValue(I.getArgOperand(0)).getValueType(),
6873 getValue(I.getArgOperand(0)),
6874 getValue(I.getArgOperand(1)), Flags));
6875 return;
6876 case Intrinsic::copysign:
6877 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
6878 getValue(I.getArgOperand(0)).getValueType(),
6879 getValue(I.getArgOperand(0)),
6880 getValue(I.getArgOperand(1)), Flags));
6881 return;
6882 case Intrinsic::ldexp:
6883 setValue(&I, DAG.getNode(ISD::FLDEXP, sdl,
6884 getValue(I.getArgOperand(0)).getValueType(),
6885 getValue(I.getArgOperand(0)),
6886 getValue(I.getArgOperand(1)), Flags));
6887 return;
6888 case Intrinsic::modf:
6889 case Intrinsic::sincos:
6890 case Intrinsic::sincospi:
6891 case Intrinsic::frexp: {
6892 unsigned Opcode;
6893 switch (Intrinsic) {
6894 default:
6895 llvm_unreachable("unexpected intrinsic");
6896 case Intrinsic::sincos:
6897 Opcode = ISD::FSINCOS;
6898 break;
6899 case Intrinsic::sincospi:
6900 Opcode = ISD::FSINCOSPI;
6901 break;
6902 case Intrinsic::modf:
6903 Opcode = ISD::FMODF;
6904 break;
6905 case Intrinsic::frexp:
6906 Opcode = ISD::FFREXP;
6907 break;
6908 }
6909 SmallVector<EVT, 2> ValueVTs;
6910 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
6911 SDVTList VTs = DAG.getVTList(ValueVTs);
6912 setValue(
6913 &I, DAG.getNode(Opcode, sdl, VTs, getValue(I.getArgOperand(0)), Flags));
6914 return;
6915 }
6916 case Intrinsic::arithmetic_fence: {
6917 setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
6918 getValue(I.getArgOperand(0)).getValueType(),
6919 getValue(I.getArgOperand(0)), Flags));
6920 return;
6921 }
6922 case Intrinsic::fma:
6923 setValue(&I, DAG.getNode(
6924 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6925 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6926 getValue(I.getArgOperand(2)), Flags));
6927 return;
6928#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
6929 case Intrinsic::INTRINSIC:
6930#include "llvm/IR/ConstrainedOps.def"
6931 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6932 return;
6933#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6934#include "llvm/IR/VPIntrinsics.def"
6935 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6936 return;
6937 case Intrinsic::fptrunc_round: {
6938 // Get the last argument, the metadata and convert it to an integer in the
6939 // call
6940 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6941 std::optional<RoundingMode> RoundMode =
6942 convertStrToRoundingMode(cast<MDString>(MD)->getString());
6943
6944 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6945
6946 // Propagate fast-math-flags from IR to node(s).
6947 SDNodeFlags Flags;
6948 Flags.copyFMF(*cast<FPMathOperator>(&I));
6949 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6950
6952 Result = DAG.getNode(
6953 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6954 DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));
6955 setValue(&I, Result);
6956
6957 return;
6958 }
6959 case Intrinsic::fmuladd: {
6960 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6961 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
6962 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
6963 setValue(&I, DAG.getNode(ISD::FMA, sdl,
6964 getValue(I.getArgOperand(0)).getValueType(),
6965 getValue(I.getArgOperand(0)),
6966 getValue(I.getArgOperand(1)),
6967 getValue(I.getArgOperand(2)), Flags));
6968 } else if (TLI.isOperationLegalOrCustom(ISD::FMULADD, VT)) {
6969 // TODO: Support splitting the vector.
6970 setValue(&I, DAG.getNode(ISD::FMULADD, sdl,
6971 getValue(I.getArgOperand(0)).getValueType(),
6972 getValue(I.getArgOperand(0)),
6973 getValue(I.getArgOperand(1)),
6974 getValue(I.getArgOperand(2)), Flags));
6975 } else {
6976 // TODO: Intrinsic calls should have fast-math-flags.
6977 SDValue Mul = DAG.getNode(
6978 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6979 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6980 SDValue Add = DAG.getNode(ISD::FADD, sdl,
6981 getValue(I.getArgOperand(0)).getValueType(),
6982 Mul, getValue(I.getArgOperand(2)), Flags);
6983 setValue(&I, Add);
6984 }
6985 return;
6986 }
6987 case Intrinsic::convert_to_fp16:
6988 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6989 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6990 getValue(I.getArgOperand(0)),
6991 DAG.getTargetConstant(0, sdl,
6992 MVT::i32))));
6993 return;
6994 case Intrinsic::convert_from_fp16:
6995 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
6996 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6997 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6998 getValue(I.getArgOperand(0)))));
6999 return;
7000 case Intrinsic::fptosi_sat: {
7001 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7002 setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
7003 getValue(I.getArgOperand(0)),
7004 DAG.getValueType(VT.getScalarType())));
7005 return;
7006 }
7007 case Intrinsic::fptoui_sat: {
7008 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7009 setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
7010 getValue(I.getArgOperand(0)),
7011 DAG.getValueType(VT.getScalarType())));
7012 return;
7013 }
7014 case Intrinsic::set_rounding:
7015 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
7016 {getRoot(), getValue(I.getArgOperand(0))});
7017 setValue(&I, Res);
7018 DAG.setRoot(Res.getValue(0));
7019 return;
7020 case Intrinsic::is_fpclass: {
7021 const DataLayout DLayout = DAG.getDataLayout();
7022 EVT DestVT = TLI.getValueType(DLayout, I.getType());
7023 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
7024 FPClassTest Test = static_cast<FPClassTest>(
7025 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
7026 MachineFunction &MF = DAG.getMachineFunction();
7027 const Function &F = MF.getFunction();
7028 SDValue Op = getValue(I.getArgOperand(0));
7029 SDNodeFlags Flags;
7030 Flags.setNoFPExcept(
7031 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7032 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7033 // expansion can use illegal types. Making expansion early allows
7034 // legalizing these types prior to selection.
7035 if (!TLI.isOperationLegal(ISD::IS_FPCLASS, ArgVT) &&
7036 !TLI.isOperationCustom(ISD::IS_FPCLASS, ArgVT)) {
7037 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
7038 setValue(&I, Result);
7039 return;
7040 }
7041
7042 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
7043 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
7044 setValue(&I, V);
7045 return;
7046 }
7047 case Intrinsic::get_fpenv: {
7048 const DataLayout DLayout = DAG.getDataLayout();
7049 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
7050 Align TempAlign = DAG.getEVTAlign(EnvVT);
7051 SDValue Chain = getRoot();
7052 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7053 // and temporary storage in stack.
7054 if (TLI.isOperationLegalOrCustom(ISD::GET_FPENV, EnvVT)) {
7055 Res = DAG.getNode(
7056 ISD::GET_FPENV, sdl,
7057 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7058 MVT::Other),
7059 Chain);
7060 } else {
7061 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7062 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7063 auto MPI =
7064 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
7065 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7067 TempAlign);
7068 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7069 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
7070 }
7071 setValue(&I, Res);
7072 DAG.setRoot(Res.getValue(1));
7073 return;
7074 }
7075 case Intrinsic::set_fpenv: {
7076 const DataLayout DLayout = DAG.getDataLayout();
7077 SDValue Env = getValue(I.getArgOperand(0));
7078 EVT EnvVT = Env.getValueType();
7079 Align TempAlign = DAG.getEVTAlign(EnvVT);
7080 SDValue Chain = getRoot();
7081 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7082 // environment from memory.
7083 if (TLI.isOperationLegalOrCustom(ISD::SET_FPENV, EnvVT)) {
7084 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
7085 } else {
7086 // Allocate space in stack, copy environment bits into it and use this
7087 // memory in SET_FPENV_MEM.
7088 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7089 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7090 auto MPI =
7091 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
7092 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7094 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7096 TempAlign);
7097 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7098 }
7099 DAG.setRoot(Chain);
7100 return;
7101 }
7102 case Intrinsic::reset_fpenv:
7103 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
7104 return;
7105 case Intrinsic::get_fpmode:
7106 Res = DAG.getNode(
7107 ISD::GET_FPMODE, sdl,
7108 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7109 MVT::Other),
7110 DAG.getRoot());
7111 setValue(&I, Res);
7112 DAG.setRoot(Res.getValue(1));
7113 return;
7114 case Intrinsic::set_fpmode:
7115 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
7116 getValue(I.getArgOperand(0)));
7117 DAG.setRoot(Res);
7118 return;
7119 case Intrinsic::reset_fpmode: {
7120 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
7121 DAG.setRoot(Res);
7122 return;
7123 }
7124 case Intrinsic::pcmarker: {
7125 SDValue Tmp = getValue(I.getArgOperand(0));
7126 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
7127 return;
7128 }
7129 case Intrinsic::readcyclecounter: {
7130 SDValue Op = getRoot();
7131 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
7132 DAG.getVTList(MVT::i64, MVT::Other), Op);
7133 setValue(&I, Res);
7134 DAG.setRoot(Res.getValue(1));
7135 return;
7136 }
7137 case Intrinsic::readsteadycounter: {
7138 SDValue Op = getRoot();
7139 Res = DAG.getNode(ISD::READSTEADYCOUNTER, sdl,
7140 DAG.getVTList(MVT::i64, MVT::Other), Op);
7141 setValue(&I, Res);
7142 DAG.setRoot(Res.getValue(1));
7143 return;
7144 }
7145 case Intrinsic::bitreverse:
7146 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
7147 getValue(I.getArgOperand(0)).getValueType(),
7148 getValue(I.getArgOperand(0))));
7149 return;
7150 case Intrinsic::bswap:
7151 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
7152 getValue(I.getArgOperand(0)).getValueType(),
7153 getValue(I.getArgOperand(0))));
7154 return;
7155 case Intrinsic::cttz: {
7156 SDValue Arg = getValue(I.getArgOperand(0));
7157 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7158 EVT Ty = Arg.getValueType();
7159 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
7160 sdl, Ty, Arg));
7161 return;
7162 }
7163 case Intrinsic::ctlz: {
7164 SDValue Arg = getValue(I.getArgOperand(0));
7165 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7166 EVT Ty = Arg.getValueType();
7167 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
7168 sdl, Ty, Arg));
7169 return;
7170 }
7171 case Intrinsic::ctpop: {
7172 SDValue Arg = getValue(I.getArgOperand(0));
7173 EVT Ty = Arg.getValueType();
7174 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7175 return;
7176 }
7177 case Intrinsic::fshl:
7178 case Intrinsic::fshr: {
7179 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7180 SDValue X = getValue(I.getArgOperand(0));
7181 SDValue Y = getValue(I.getArgOperand(1));
7182 SDValue Z = getValue(I.getArgOperand(2));
7183 EVT VT = X.getValueType();
7184
7185 if (X == Y) {
7186 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7187 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7188 } else {
7189 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7190 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7191 }
7192 return;
7193 }
7194 case Intrinsic::sadd_sat: {
7195 SDValue Op1 = getValue(I.getArgOperand(0));
7196 SDValue Op2 = getValue(I.getArgOperand(1));
7197 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7198 return;
7199 }
7200 case Intrinsic::uadd_sat: {
7201 SDValue Op1 = getValue(I.getArgOperand(0));
7202 SDValue Op2 = getValue(I.getArgOperand(1));
7203 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7204 return;
7205 }
7206 case Intrinsic::ssub_sat: {
7207 SDValue Op1 = getValue(I.getArgOperand(0));
7208 SDValue Op2 = getValue(I.getArgOperand(1));
7209 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7210 return;
7211 }
7212 case Intrinsic::usub_sat: {
7213 SDValue Op1 = getValue(I.getArgOperand(0));
7214 SDValue Op2 = getValue(I.getArgOperand(1));
7215 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7216 return;
7217 }
7218 case Intrinsic::sshl_sat: {
7219 SDValue Op1 = getValue(I.getArgOperand(0));
7220 SDValue Op2 = getValue(I.getArgOperand(1));
7221 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7222 return;
7223 }
7224 case Intrinsic::ushl_sat: {
7225 SDValue Op1 = getValue(I.getArgOperand(0));
7226 SDValue Op2 = getValue(I.getArgOperand(1));
7227 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7228 return;
7229 }
7230 case Intrinsic::smul_fix:
7231 case Intrinsic::umul_fix:
7232 case Intrinsic::smul_fix_sat:
7233 case Intrinsic::umul_fix_sat: {
7234 SDValue Op1 = getValue(I.getArgOperand(0));
7235 SDValue Op2 = getValue(I.getArgOperand(1));
7236 SDValue Op3 = getValue(I.getArgOperand(2));
7237 setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
7238 Op1.getValueType(), Op1, Op2, Op3));
7239 return;
7240 }
7241 case Intrinsic::sdiv_fix:
7242 case Intrinsic::udiv_fix:
7243 case Intrinsic::sdiv_fix_sat:
7244 case Intrinsic::udiv_fix_sat: {
7245 SDValue Op1 = getValue(I.getArgOperand(0));
7246 SDValue Op2 = getValue(I.getArgOperand(1));
7247 SDValue Op3 = getValue(I.getArgOperand(2));
7249 Op1, Op2, Op3, DAG, TLI));
7250 return;
7251 }
7252 case Intrinsic::smax: {
7253 SDValue Op1 = getValue(I.getArgOperand(0));
7254 SDValue Op2 = getValue(I.getArgOperand(1));
7255 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7256 return;
7257 }
7258 case Intrinsic::smin: {
7259 SDValue Op1 = getValue(I.getArgOperand(0));
7260 SDValue Op2 = getValue(I.getArgOperand(1));
7261 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7262 return;
7263 }
7264 case Intrinsic::umax: {
7265 SDValue Op1 = getValue(I.getArgOperand(0));
7266 SDValue Op2 = getValue(I.getArgOperand(1));
7267 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7268 return;
7269 }
7270 case Intrinsic::umin: {
7271 SDValue Op1 = getValue(I.getArgOperand(0));
7272 SDValue Op2 = getValue(I.getArgOperand(1));
7273 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7274 return;
7275 }
7276 case Intrinsic::abs: {
7277 // TODO: Preserve "int min is poison" arg in SDAG?
7278 SDValue Op1 = getValue(I.getArgOperand(0));
7279 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
7280 return;
7281 }
7282 case Intrinsic::scmp: {
7283 SDValue Op1 = getValue(I.getArgOperand(0));
7284 SDValue Op2 = getValue(I.getArgOperand(1));
7285 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7286 setValue(&I, DAG.getNode(ISD::SCMP, sdl, DestVT, Op1, Op2));
7287 break;
7288 }
7289 case Intrinsic::ucmp: {
7290 SDValue Op1 = getValue(I.getArgOperand(0));
7291 SDValue Op2 = getValue(I.getArgOperand(1));
7292 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7293 setValue(&I, DAG.getNode(ISD::UCMP, sdl, DestVT, Op1, Op2));
7294 break;
7295 }
7296 case Intrinsic::stacksave: {
7297 SDValue Op = getRoot();
7298 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7299 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
7300 setValue(&I, Res);
7301 DAG.setRoot(Res.getValue(1));
7302 return;
7303 }
7304 case Intrinsic::stackrestore:
7305 Res = getValue(I.getArgOperand(0));
7306 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7307 return;
7308 case Intrinsic::get_dynamic_area_offset: {
7309 SDValue Op = getRoot();
7310 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7311 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
7312 Op);
7313 DAG.setRoot(Op);
7314 setValue(&I, Res);
7315 return;
7316 }
7317 case Intrinsic::stackguard: {
7318 MachineFunction &MF = DAG.getMachineFunction();
7319 const Module &M = *MF.getFunction().getParent();
7320 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7321 SDValue Chain = getRoot();
7322 if (TLI.useLoadStackGuardNode(M)) {
7323 Res = getLoadStackGuard(DAG, sdl, Chain);
7324 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7325 } else {
7326 const Value *Global = TLI.getSDagStackGuard(M);
7327 Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
7328 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7329 MachinePointerInfo(Global, 0), Align,
7331 }
7332 if (TLI.useStackGuardXorFP())
7333 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
7334 DAG.setRoot(Chain);
7335 setValue(&I, Res);
7336 return;
7337 }
7338 case Intrinsic::stackprotector: {
7339 // Emit code into the DAG to store the stack guard onto the stack.
7340 MachineFunction &MF = DAG.getMachineFunction();
7341 MachineFrameInfo &MFI = MF.getFrameInfo();
7342 const Module &M = *MF.getFunction().getParent();
7343 SDValue Src, Chain = getRoot();
7344
7345 if (TLI.useLoadStackGuardNode(M))
7346 Src = getLoadStackGuard(DAG, sdl, Chain);
7347 else
7348 Src = getValue(I.getArgOperand(0)); // The guard's value.
7349
7350 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7351
7352 int FI = FuncInfo.StaticAllocaMap[Slot];
7353 MFI.setStackProtectorIndex(FI);
7354 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7355
7356 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7357
7358 // Store the stack protector onto the stack.
7359 Res = DAG.getStore(
7360 Chain, sdl, Src, FIN,
7361 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
7362 MaybeAlign(), MachineMemOperand::MOVolatile);
7363 setValue(&I, Res);
7364 DAG.setRoot(Res);
7365 return;
7366 }
7367 case Intrinsic::objectsize:
7368 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7369
7370 case Intrinsic::is_constant:
7371 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7372
7373 case Intrinsic::annotation:
7374 case Intrinsic::ptr_annotation:
7375 case Intrinsic::launder_invariant_group:
7376 case Intrinsic::strip_invariant_group:
7377 // Drop the intrinsic, but forward the value
7378 setValue(&I, getValue(I.getOperand(0)));
7379 return;
7380
7381 case Intrinsic::type_test:
7382 case Intrinsic::public_type_test:
7383 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
7384 return;
7385
7386 case Intrinsic::assume:
7387 case Intrinsic::experimental_noalias_scope_decl:
7388 case Intrinsic::var_annotation:
7389 case Intrinsic::sideeffect:
7390 // Discard annotate attributes, noalias scope declarations, assumptions, and
7391 // artificial side-effects.
7392 return;
7393
7394 case Intrinsic::codeview_annotation: {
7395 // Emit a label associated with this metadata.
7396 MachineFunction &MF = DAG.getMachineFunction();
7397 MCSymbol *Label = MF.getContext().createTempSymbol("annotation", true);
7398 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7399 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7400 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7401 DAG.setRoot(Res);
7402 return;
7403 }
7404
7405 case Intrinsic::init_trampoline: {
7406 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7407
7408 SDValue Ops[6];
7409 Ops[0] = getRoot();
7410 Ops[1] = getValue(I.getArgOperand(0));
7411 Ops[2] = getValue(I.getArgOperand(1));
7412 Ops[3] = getValue(I.getArgOperand(2));
7413 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7414 Ops[5] = DAG.getSrcValue(F);
7415
7416 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7417
7418 DAG.setRoot(Res);
7419 return;
7420 }
7421 case Intrinsic::adjust_trampoline:
7422 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
7423 TLI.getPointerTy(DAG.getDataLayout()),
7424 getValue(I.getArgOperand(0))));
7425 return;
7426 case Intrinsic::gcroot: {
7427 assert(DAG.getMachineFunction().getFunction().hasGC() &&
7428 "only valid in functions with gc specified, enforced by Verifier");
7429 assert(GFI && "implied by previous");
7430 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7431 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7432
7433 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7434 GFI->addStackRoot(FI->getIndex(), TypeMap);
7435 return;
7436 }
7437 case Intrinsic::gcread:
7438 case Intrinsic::gcwrite:
7439 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7440 case Intrinsic::get_rounding:
7441 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7442 setValue(&I, Res);
7443 DAG.setRoot(Res.getValue(1));
7444 return;
7445
7446 case Intrinsic::expect:
7447 case Intrinsic::expect_with_probability:
7448 // Just replace __builtin_expect(exp, c) and
7449 // __builtin_expect_with_probability(exp, c, p) with EXP.
7450 setValue(&I, getValue(I.getArgOperand(0)));
7451 return;
7452
7453 case Intrinsic::ubsantrap:
7454 case Intrinsic::debugtrap:
7455 case Intrinsic::trap: {
7456 StringRef TrapFuncName =
7457 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
7458 if (TrapFuncName.empty()) {
7459 switch (Intrinsic) {
7460 case Intrinsic::trap:
7461 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
7462 break;
7463 case Intrinsic::debugtrap:
7464 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
7465 break;
7466 case Intrinsic::ubsantrap:
7467 DAG.setRoot(DAG.getNode(
7468 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
7469 DAG.getTargetConstant(
7470 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
7471 MVT::i32)));
7472 break;
7473 default: llvm_unreachable("unknown trap intrinsic");
7474 }
7475 DAG.addNoMergeSiteInfo(DAG.getRoot().getNode(),
7476 I.hasFnAttr(Attribute::NoMerge));
7477 return;
7478 }
7480 if (Intrinsic == Intrinsic::ubsantrap) {
7481 Value *Arg = I.getArgOperand(0);
7482 Args.emplace_back(Arg, getValue(Arg));
7483 }
7484
7485 TargetLowering::CallLoweringInfo CLI(DAG);
7486 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7487 CallingConv::C, I.getType(),
7488 DAG.getExternalSymbol(TrapFuncName.data(),
7489 TLI.getPointerTy(DAG.getDataLayout())),
7490 std::move(Args));
7491 CLI.NoMerge = I.hasFnAttr(Attribute::NoMerge);
7492 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7493 DAG.setRoot(Result.second);
7494 return;
7495 }
7496
7497 case Intrinsic::allow_runtime_check:
7498 case Intrinsic::allow_ubsan_check:
7499 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
7500 return;
7501
7502 case Intrinsic::uadd_with_overflow:
7503 case Intrinsic::sadd_with_overflow:
7504 case Intrinsic::usub_with_overflow:
7505 case Intrinsic::ssub_with_overflow:
7506 case Intrinsic::umul_with_overflow:
7507 case Intrinsic::smul_with_overflow: {
7509 switch (Intrinsic) {
7510 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7511 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7512 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7513 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7514 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7515 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7516 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7517 }
7518 SDValue Op1 = getValue(I.getArgOperand(0));
7519 SDValue Op2 = getValue(I.getArgOperand(1));
7520
7521 EVT ResultVT = Op1.getValueType();
7522 EVT OverflowVT = MVT::i1;
7523 if (ResultVT.isVector())
7524 OverflowVT = EVT::getVectorVT(
7525 *Context, OverflowVT, ResultVT.getVectorElementCount());
7526
7527 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
7528 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
7529 return;
7530 }
7531 case Intrinsic::prefetch: {
7532 SDValue Ops[5];
7533 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7535 Ops[0] = DAG.getRoot();
7536 Ops[1] = getValue(I.getArgOperand(0));
7537 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7538 MVT::i32);
7539 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
7540 MVT::i32);
7541 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
7542 MVT::i32);
7543 SDValue Result = DAG.getMemIntrinsicNode(
7544 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
7545 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
7546 /* align */ std::nullopt, Flags);
7547
7548 // Chain the prefetch in parallel with any pending loads, to stay out of
7549 // the way of later optimizations.
7550 PendingLoads.push_back(Result);
7551 Result = getRoot();
7552 DAG.setRoot(Result);
7553 return;
7554 }
7555 case Intrinsic::lifetime_start:
7556 case Intrinsic::lifetime_end: {
7557 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7558 // Stack coloring is not enabled in O0, discard region information.
7559 if (TM.getOptLevel() == CodeGenOptLevel::None)
7560 return;
7561
7562 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(I.getArgOperand(0));
7563 if (!LifetimeObject)
7564 return;
7565
7566 // First check that the Alloca is static, otherwise it won't have a
7567 // valid frame index.
7568 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
7569 if (SI == FuncInfo.StaticAllocaMap.end())
7570 return;
7571
7572 const int FrameIndex = SI->second;
7573 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex);
7574 DAG.setRoot(Res);
7575 return;
7576 }
7577 case Intrinsic::pseudoprobe: {
7578 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
7579 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7580 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
7581 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
7582 DAG.setRoot(Res);
7583 return;
7584 }
7585 case Intrinsic::invariant_start:
7586 // Discard region information.
7587 setValue(&I,
7588 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
7589 return;
7590 case Intrinsic::invariant_end:
7591 // Discard region information.
7592 return;
7593 case Intrinsic::clear_cache: {
7594 SDValue InputChain = DAG.getRoot();
7595 SDValue StartVal = getValue(I.getArgOperand(0));
7596 SDValue EndVal = getValue(I.getArgOperand(1));
7597 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),
7598 {InputChain, StartVal, EndVal});
7599 setValue(&I, Res);
7600 DAG.setRoot(Res);
7601 return;
7602 }
7603 case Intrinsic::donothing:
7604 case Intrinsic::seh_try_begin:
7605 case Intrinsic::seh_scope_begin:
7606 case Intrinsic::seh_try_end:
7607 case Intrinsic::seh_scope_end:
7608 // ignore
7609 return;
7610 case Intrinsic::experimental_stackmap:
7611 visitStackmap(I);
7612 return;
7613 case Intrinsic::experimental_patchpoint_void:
7614 case Intrinsic::experimental_patchpoint:
7615 visitPatchpoint(I);
7616 return;
7617 case Intrinsic::experimental_gc_statepoint:
7619 return;
7620 case Intrinsic::experimental_gc_result:
7621 visitGCResult(cast<GCResultInst>(I));
7622 return;
7623 case Intrinsic::experimental_gc_relocate:
7624 visitGCRelocate(cast<GCRelocateInst>(I));
7625 return;
7626 case Intrinsic::instrprof_cover:
7627 llvm_unreachable("instrprof failed to lower a cover");
7628 case Intrinsic::instrprof_increment:
7629 llvm_unreachable("instrprof failed to lower an increment");
7630 case Intrinsic::instrprof_timestamp:
7631 llvm_unreachable("instrprof failed to lower a timestamp");
7632 case Intrinsic::instrprof_value_profile:
7633 llvm_unreachable("instrprof failed to lower a value profiling call");
7634 case Intrinsic::instrprof_mcdc_parameters:
7635 llvm_unreachable("instrprof failed to lower mcdc parameters");
7636 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7637 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7638 case Intrinsic::localescape: {
7639 MachineFunction &MF = DAG.getMachineFunction();
7640 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
7641
7642 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7643 // is the same on all targets.
7644 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7645 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
7646 if (isa<ConstantPointerNull>(Arg))
7647 continue; // Skip null pointers. They represent a hole in index space.
7648 AllocaInst *Slot = cast<AllocaInst>(Arg);
7649 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7650 "can only escape static allocas");
7651 int FI = FuncInfo.StaticAllocaMap[Slot];
7652 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7654 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
7655 TII->get(TargetOpcode::LOCAL_ESCAPE))
7656 .addSym(FrameAllocSym)
7657 .addFrameIndex(FI);
7658 }
7659
7660 return;
7661 }
7662
7663 case Intrinsic::localrecover: {
7664 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7665 MachineFunction &MF = DAG.getMachineFunction();
7666
7667 // Get the symbol that defines the frame offset.
7668 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
7669 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
7670 unsigned IdxVal =
7671 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
7672 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7674
7675 Value *FP = I.getArgOperand(1);
7676 SDValue FPVal = getValue(FP);
7677 EVT PtrVT = FPVal.getValueType();
7678
7679 // Create a MCSymbol for the label to avoid any target lowering
7680 // that would make this PC relative.
7681 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
7682 SDValue OffsetVal =
7683 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
7684
7685 // Add the offset to the FP.
7686 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
7687 setValue(&I, Add);
7688
7689 return;
7690 }
7691
7692 case Intrinsic::fake_use: {
7693 Value *V = I.getArgOperand(0);
7694 SDValue Ops[2];
7695 // For Values not declared or previously used in this basic block, the
7696 // NodeMap will not have an entry, and `getValue` will assert if V has no
7697 // valid register value.
7698 auto FakeUseValue = [&]() -> SDValue {
7699 SDValue &N = NodeMap[V];
7700 if (N.getNode())
7701 return N;
7702
7703 // If there's a virtual register allocated and initialized for this
7704 // value, use it.
7705 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
7706 return copyFromReg;
7707 // FIXME: Do we want to preserve constants? It seems pointless.
7708 if (isa<Constant>(V))
7709 return getValue(V);
7710 return SDValue();
7711 }();
7712 if (!FakeUseValue || FakeUseValue.isUndef())
7713 return;
7714 Ops[0] = getRoot();
7715 Ops[1] = FakeUseValue;
7716 // Also, do not translate a fake use with an undef operand, or any other
7717 // empty SDValues.
7718 if (!Ops[1] || Ops[1].isUndef())
7719 return;
7720 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other, Ops));
7721 return;
7722 }
7723
7724 case Intrinsic::eh_exceptionpointer:
7725 case Intrinsic::eh_exceptioncode: {
7726 // Get the exception pointer vreg, copy from it, and resize it to fit.
7727 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7728 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7729 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7730 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7731 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7732 if (Intrinsic == Intrinsic::eh_exceptioncode)
7733 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7734 setValue(&I, N);
7735 return;
7736 }
7737 case Intrinsic::xray_customevent: {
7738 // Here we want to make sure that the intrinsic behaves as if it has a
7739 // specific calling convention.
7740 const auto &Triple = DAG.getTarget().getTargetTriple();
7741 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7742 return;
7743
7745
7746 // We want to say that we always want the arguments in registers.
7747 SDValue LogEntryVal = getValue(I.getArgOperand(0));
7748 SDValue StrSizeVal = getValue(I.getArgOperand(1));
7749 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7750 SDValue Chain = getRoot();
7751 Ops.push_back(LogEntryVal);
7752 Ops.push_back(StrSizeVal);
7753 Ops.push_back(Chain);
7754
7755 // We need to enforce the calling convention for the callsite, so that
7756 // argument ordering is enforced correctly, and that register allocation can
7757 // see that some registers may be assumed clobbered and have to preserve
7758 // them across calls to the intrinsic.
7759 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7760 sdl, NodeTys, Ops);
7761 SDValue patchableNode = SDValue(MN, 0);
7762 DAG.setRoot(patchableNode);
7763 setValue(&I, patchableNode);
7764 return;
7765 }
7766 case Intrinsic::xray_typedevent: {
7767 // Here we want to make sure that the intrinsic behaves as if it has a
7768 // specific calling convention.
7769 const auto &Triple = DAG.getTarget().getTargetTriple();
7770 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7771 return;
7772
7774
7775 // We want to say that we always want the arguments in registers.
7776 // It's unclear to me how manipulating the selection DAG here forces callers
7777 // to provide arguments in registers instead of on the stack.
7778 SDValue LogTypeId = getValue(I.getArgOperand(0));
7779 SDValue LogEntryVal = getValue(I.getArgOperand(1));
7780 SDValue StrSizeVal = getValue(I.getArgOperand(2));
7781 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7782 SDValue Chain = getRoot();
7783 Ops.push_back(LogTypeId);
7784 Ops.push_back(LogEntryVal);
7785 Ops.push_back(StrSizeVal);
7786 Ops.push_back(Chain);
7787
7788 // We need to enforce the calling convention for the callsite, so that
7789 // argument ordering is enforced correctly, and that register allocation can
7790 // see that some registers may be assumed clobbered and have to preserve
7791 // them across calls to the intrinsic.
7792 MachineSDNode *MN = DAG.getMachineNode(
7793 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7794 SDValue patchableNode = SDValue(MN, 0);
7795 DAG.setRoot(patchableNode);
7796 setValue(&I, patchableNode);
7797 return;
7798 }
7799 case Intrinsic::experimental_deoptimize:
7801 return;
7802 case Intrinsic::stepvector:
7803 visitStepVector(I);
7804 return;
7805 case Intrinsic::vector_reduce_fadd:
7806 case Intrinsic::vector_reduce_fmul:
7807 case Intrinsic::vector_reduce_add:
7808 case Intrinsic::vector_reduce_mul:
7809 case Intrinsic::vector_reduce_and:
7810 case Intrinsic::vector_reduce_or:
7811 case Intrinsic::vector_reduce_xor:
7812 case Intrinsic::vector_reduce_smax:
7813 case Intrinsic::vector_reduce_smin:
7814 case Intrinsic::vector_reduce_umax:
7815 case Intrinsic::vector_reduce_umin:
7816 case Intrinsic::vector_reduce_fmax:
7817 case Intrinsic::vector_reduce_fmin:
7818 case Intrinsic::vector_reduce_fmaximum:
7819 case Intrinsic::vector_reduce_fminimum:
7820 visitVectorReduce(I, Intrinsic);
7821 return;
7822
7823 case Intrinsic::icall_branch_funnel: {
7825 Ops.push_back(getValue(I.getArgOperand(0)));
7826
7827 int64_t Offset;
7829 I.getArgOperand(1), Offset, DAG.getDataLayout()));
7830 if (!Base)
7832 "llvm.icall.branch.funnel operand must be a GlobalValue");
7833 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7834
7835 struct BranchFunnelTarget {
7836 int64_t Offset;
7838 };
7840
7841 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7843 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7844 if (ElemBase != Base)
7845 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7846 "to the same GlobalValue");
7847
7848 SDValue Val = getValue(I.getArgOperand(Op + 1));
7849 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7850 if (!GA)
7852 "llvm.icall.branch.funnel operand must be a GlobalValue");
7853 Targets.push_back({Offset, DAG.getTargetGlobalAddress(
7854 GA->getGlobal(), sdl, Val.getValueType(),
7855 GA->getOffset())});
7856 }
7857 llvm::sort(Targets,
7858 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7859 return T1.Offset < T2.Offset;
7860 });
7861
7862 for (auto &T : Targets) {
7863 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7864 Ops.push_back(T.Target);
7865 }
7866
7867 Ops.push_back(DAG.getRoot()); // Chain
7868 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7869 MVT::Other, Ops),
7870 0);
7871 DAG.setRoot(N);
7872 setValue(&I, N);
7873 HasTailCall = true;
7874 return;
7875 }
7876
7877 case Intrinsic::wasm_landingpad_index:
7878 // Information this intrinsic contained has been transferred to
7879 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7880 // delete it now.
7881 return;
7882
7883 case Intrinsic::aarch64_settag:
7884 case Intrinsic::aarch64_settag_zero: {
7885 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
7886 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7888 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7889 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7890 ZeroMemory);
7891 DAG.setRoot(Val);
7892 setValue(&I, Val);
7893 return;
7894 }
7895 case Intrinsic::amdgcn_cs_chain: {
7896 // At this point we don't care if it's amdgpu_cs_chain or
7897 // amdgpu_cs_chain_preserve.
7899
7900 Type *RetTy = I.getType();
7901 assert(RetTy->isVoidTy() && "Should not return");
7902
7903 SDValue Callee = getValue(I.getOperand(0));
7904
7905 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
7906 // We'll also tack the value of the EXEC mask at the end.
7908 Args.reserve(3);
7909
7910 for (unsigned Idx : {2, 3, 1}) {
7911 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
7912 I.getOperand(Idx)->getType());
7913 Arg.setAttributes(&I, Idx);
7914 Args.push_back(Arg);
7915 }
7916
7917 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
7918 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
7919 Args[2].IsInReg = true; // EXEC should be inreg
7920
7921 // Forward the flags and any additional arguments.
7922 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
7923 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
7924 I.getOperand(Idx)->getType());
7925 Arg.setAttributes(&I, Idx);
7926 Args.push_back(Arg);
7927 }
7928
7929 TargetLowering::CallLoweringInfo CLI(DAG);
7930 CLI.setDebugLoc(getCurSDLoc())
7931 .setChain(getRoot())
7932 .setCallee(CC, RetTy, Callee, std::move(Args))
7933 .setNoReturn(true)
7934 .setTailCall(true)
7935 .setConvergent(I.isConvergent());
7936 CLI.CB = &I;
7937 std::pair<SDValue, SDValue> Result =
7938 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
7939 (void)Result;
7940 assert(!Result.first.getNode() && !Result.second.getNode() &&
7941 "Should've lowered as tail call");
7942
7943 HasTailCall = true;
7944 return;
7945 }
7946 case Intrinsic::amdgcn_call_whole_wave: {
7948 bool isTailCall = I.isTailCall();
7949
7950 // The first argument is the callee. Skip it when assembling the call args.
7951 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
7952 TargetLowering::ArgListEntry Arg(getValue(I.getArgOperand(Idx)),
7953 I.getArgOperand(Idx)->getType());
7954 Arg.setAttributes(&I, Idx);
7955
7956 // If we have an explicit sret argument that is an Instruction, (i.e., it
7957 // might point to function-local memory), we can't meaningfully tail-call.
7958 if (Arg.IsSRet && isa<Instruction>(I.getArgOperand(Idx)))
7959 isTailCall = false;
7960
7961 Args.push_back(Arg);
7962 }
7963
7964 SDValue ConvControlToken;
7965 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
7966 auto *Token = Bundle->Inputs[0].get();
7967 ConvControlToken = getValue(Token);
7968 }
7969
7970 TargetLowering::CallLoweringInfo CLI(DAG);
7971 CLI.setDebugLoc(getCurSDLoc())
7972 .setChain(getRoot())
7973 .setCallee(CallingConv::AMDGPU_Gfx_WholeWave, I.getType(),
7974 getValue(I.getArgOperand(0)), std::move(Args))
7975 .setTailCall(isTailCall && canTailCall(I))
7976 .setIsPreallocated(
7977 I.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
7978 .setConvergent(I.isConvergent())
7979 .setConvergenceControlToken(ConvControlToken);
7980 CLI.CB = &I;
7981
7982 std::pair<SDValue, SDValue> Result =
7983 lowerInvokable(CLI, /*EHPadBB=*/nullptr);
7984
7985 if (Result.first.getNode())
7986 setValue(&I, Result.first);
7987 return;
7988 }
7989 case Intrinsic::ptrmask: {
7990 SDValue Ptr = getValue(I.getOperand(0));
7991 SDValue Mask = getValue(I.getOperand(1));
7992
7993 // On arm64_32, pointers are 32 bits when stored in memory, but
7994 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
7995 // match the index type, but the pointer is 64 bits, so the mask must be
7996 // zero-extended up to 64 bits to match the pointer.
7997 EVT PtrVT =
7998 TLI.getValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
7999 EVT MemVT =
8000 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8001 assert(PtrVT == Ptr.getValueType());
8002 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
8003 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
8004 // 128-bit, so we have to pad the mask with ones for unused bits.
8005 auto HighOnes = DAG.getNode(
8006 ISD::SHL, sdl, PtrVT, DAG.getAllOnesConstant(sdl, PtrVT),
8007 DAG.getShiftAmountConstant(Mask.getValueType().getFixedSizeInBits(),
8008 PtrVT, sdl));
8009 Mask = DAG.getNode(ISD::OR, sdl, PtrVT,
8010 DAG.getZExtOrTrunc(Mask, sdl, PtrVT), HighOnes);
8011 } else if (Mask.getValueType() != PtrVT)
8012 Mask = DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);
8013
8014 assert(Mask.getValueType() == PtrVT);
8015 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
8016 return;
8017 }
8018 case Intrinsic::threadlocal_address: {
8019 setValue(&I, getValue(I.getOperand(0)));
8020 return;
8021 }
8022 case Intrinsic::get_active_lane_mask: {
8023 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8024 SDValue Index = getValue(I.getOperand(0));
8025 SDValue TripCount = getValue(I.getOperand(1));
8026 EVT ElementVT = Index.getValueType();
8027
8028 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
8029 setValue(&I, DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, sdl, CCVT, Index,
8030 TripCount));
8031 return;
8032 }
8033
8034 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
8035 CCVT.getVectorElementCount());
8036
8037 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
8038 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
8039 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
8040 SDValue VectorInduction = DAG.getNode(
8041 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
8042 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
8043 VectorTripCount, ISD::CondCode::SETULT);
8044 setValue(&I, SetCC);
8045 return;
8046 }
8047 case Intrinsic::experimental_get_vector_length: {
8048 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8049 "Expected positive VF");
8050 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
8051 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
8052
8053 SDValue Count = getValue(I.getOperand(0));
8054 EVT CountVT = Count.getValueType();
8055
8056 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8057 visitTargetIntrinsic(I, Intrinsic);
8058 return;
8059 }
8060
8061 // Expand to a umin between the trip count and the maximum elements the type
8062 // can hold.
8063 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8064
8065 // Extend the trip count to at least the result VT.
8066 if (CountVT.bitsLT(VT)) {
8067 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
8068 CountVT = VT;
8069 }
8070
8071 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
8072 ElementCount::get(VF, IsScalable));
8073
8074 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
8075 // Clip to the result type if needed.
8076 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
8077
8078 setValue(&I, Trunc);
8079 return;
8080 }
8081 case Intrinsic::vector_partial_reduce_add: {
8082 SDValue Acc = getValue(I.getOperand(0));
8083 SDValue Input = getValue(I.getOperand(1));
8084 setValue(&I,
8085 DAG.getNode(ISD::PARTIAL_REDUCE_UMLA, sdl, Acc.getValueType(), Acc,
8086 Input, DAG.getConstant(1, sdl, Input.getValueType())));
8087 return;
8088 }
8089 case Intrinsic::experimental_cttz_elts: {
8090 auto DL = getCurSDLoc();
8091 SDValue Op = getValue(I.getOperand(0));
8092 EVT OpVT = Op.getValueType();
8093
8094 if (!TLI.shouldExpandCttzElements(OpVT)) {
8095 visitTargetIntrinsic(I, Intrinsic);
8096 return;
8097 }
8098
8099 if (OpVT.getScalarType() != MVT::i1) {
8100 // Compare the input vector elements to zero & use to count trailing zeros
8101 SDValue AllZero = DAG.getConstant(0, DL, OpVT);
8102 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
8103 OpVT.getVectorElementCount());
8104 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
8105 }
8106
8107 // If the zero-is-poison flag is set, we can assume the upper limit
8108 // of the result is VF-1.
8109 bool ZeroIsPoison =
8110 !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
8111 ConstantRange VScaleRange(1, true); // Dummy value.
8112 if (isa<ScalableVectorType>(I.getOperand(0)->getType()))
8113 VScaleRange = getVScaleRange(I.getCaller(), 64);
8114 unsigned EltWidth = TLI.getBitWidthForCttzElements(
8115 I.getType(), OpVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
8116
8117 MVT NewEltTy = MVT::getIntegerVT(EltWidth);
8118
8119 // Create the new vector type & get the vector length
8120 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,
8121 OpVT.getVectorElementCount());
8122
8123 SDValue VL =
8124 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());
8125
8126 SDValue StepVec = DAG.getStepVector(DL, NewVT);
8127 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
8128 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
8129 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, Op);
8130 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
8131 SDValue Max = DAG.getNode(ISD::VECREDUCE_UMAX, DL, NewEltTy, And);
8132 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
8133
8134 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
8135 SDValue Ret = DAG.getZExtOrTrunc(Sub, DL, RetTy);
8136
8137 setValue(&I, Ret);
8138 return;
8139 }
8140 case Intrinsic::vector_insert: {
8141 SDValue Vec = getValue(I.getOperand(0));
8142 SDValue SubVec = getValue(I.getOperand(1));
8143 SDValue Index = getValue(I.getOperand(2));
8144
8145 // The intrinsic's index type is i64, but the SDNode requires an index type
8146 // suitable for the target. Convert the index as required.
8147 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8148 if (Index.getValueType() != VectorIdxTy)
8149 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8150
8151 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8152 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
8153 Index));
8154 return;
8155 }
8156 case Intrinsic::vector_extract: {
8157 SDValue Vec = getValue(I.getOperand(0));
8158 SDValue Index = getValue(I.getOperand(1));
8159 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8160
8161 // The intrinsic's index type is i64, but the SDNode requires an index type
8162 // suitable for the target. Convert the index as required.
8163 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8164 if (Index.getValueType() != VectorIdxTy)
8165 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8166
8167 setValue(&I,
8168 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
8169 return;
8170 }
8171 case Intrinsic::experimental_vector_match: {
8172 SDValue Op1 = getValue(I.getOperand(0));
8173 SDValue Op2 = getValue(I.getOperand(1));
8174 SDValue Mask = getValue(I.getOperand(2));
8175 EVT Op1VT = Op1.getValueType();
8176 EVT Op2VT = Op2.getValueType();
8177 EVT ResVT = Mask.getValueType();
8178 unsigned SearchSize = Op2VT.getVectorNumElements();
8179
8180 // If the target has native support for this vector match operation, lower
8181 // the intrinsic untouched; otherwise, expand it below.
8182 if (!TLI.shouldExpandVectorMatch(Op1VT, SearchSize)) {
8183 visitTargetIntrinsic(I, Intrinsic);
8184 return;
8185 }
8186
8187 SDValue Ret = DAG.getConstant(0, sdl, ResVT);
8188
8189 for (unsigned i = 0; i < SearchSize; ++i) {
8190 SDValue Op2Elem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl,
8191 Op2VT.getVectorElementType(), Op2,
8192 DAG.getVectorIdxConstant(i, sdl));
8193 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, sdl, Op1VT, Op2Elem);
8194 SDValue Cmp = DAG.getSetCC(sdl, ResVT, Op1, Splat, ISD::SETEQ);
8195 Ret = DAG.getNode(ISD::OR, sdl, ResVT, Ret, Cmp);
8196 }
8197
8198 setValue(&I, DAG.getNode(ISD::AND, sdl, ResVT, Ret, Mask));
8199 return;
8200 }
8201 case Intrinsic::vector_reverse:
8202 visitVectorReverse(I);
8203 return;
8204 case Intrinsic::vector_splice:
8205 visitVectorSplice(I);
8206 return;
8207 case Intrinsic::callbr_landingpad:
8208 visitCallBrLandingPad(I);
8209 return;
8210 case Intrinsic::vector_interleave2:
8211 visitVectorInterleave(I, 2);
8212 return;
8213 case Intrinsic::vector_interleave3:
8214 visitVectorInterleave(I, 3);
8215 return;
8216 case Intrinsic::vector_interleave4:
8217 visitVectorInterleave(I, 4);
8218 return;
8219 case Intrinsic::vector_interleave5:
8220 visitVectorInterleave(I, 5);
8221 return;
8222 case Intrinsic::vector_interleave6:
8223 visitVectorInterleave(I, 6);
8224 return;
8225 case Intrinsic::vector_interleave7:
8226 visitVectorInterleave(I, 7);
8227 return;
8228 case Intrinsic::vector_interleave8:
8229 visitVectorInterleave(I, 8);
8230 return;
8231 case Intrinsic::vector_deinterleave2:
8232 visitVectorDeinterleave(I, 2);
8233 return;
8234 case Intrinsic::vector_deinterleave3:
8235 visitVectorDeinterleave(I, 3);
8236 return;
8237 case Intrinsic::vector_deinterleave4:
8238 visitVectorDeinterleave(I, 4);
8239 return;
8240 case Intrinsic::vector_deinterleave5:
8241 visitVectorDeinterleave(I, 5);
8242 return;
8243 case Intrinsic::vector_deinterleave6:
8244 visitVectorDeinterleave(I, 6);
8245 return;
8246 case Intrinsic::vector_deinterleave7:
8247 visitVectorDeinterleave(I, 7);
8248 return;
8249 case Intrinsic::vector_deinterleave8:
8250 visitVectorDeinterleave(I, 8);
8251 return;
8252 case Intrinsic::experimental_vector_compress:
8253 setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,
8254 getValue(I.getArgOperand(0)).getValueType(),
8255 getValue(I.getArgOperand(0)),
8256 getValue(I.getArgOperand(1)),
8257 getValue(I.getArgOperand(2)), Flags));
8258 return;
8259 case Intrinsic::experimental_convergence_anchor:
8260 case Intrinsic::experimental_convergence_entry:
8261 case Intrinsic::experimental_convergence_loop:
8262 visitConvergenceControl(I, Intrinsic);
8263 return;
8264 case Intrinsic::experimental_vector_histogram_add: {
8265 visitVectorHistogram(I, Intrinsic);
8266 return;
8267 }
8268 case Intrinsic::experimental_vector_extract_last_active: {
8269 visitVectorExtractLastActive(I, Intrinsic);
8270 return;
8271 }
8272 case Intrinsic::loop_dependence_war_mask:
8273 setValue(&I,
8275 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),
8276 getValue(I.getOperand(1)), getValue(I.getOperand(2))));
8277 return;
8278 case Intrinsic::loop_dependence_raw_mask:
8279 setValue(&I,
8281 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),
8282 getValue(I.getOperand(1)), getValue(I.getOperand(2))));
8283 return;
8284 }
8285}
8286
8287void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8288 const ConstrainedFPIntrinsic &FPI) {
8289 SDLoc sdl = getCurSDLoc();
8290
8291 // We do not need to serialize constrained FP intrinsics against
8292 // each other or against (nonvolatile) loads, so they can be
8293 // chained like loads.
8294 SDValue Chain = DAG.getRoot();
8296 Opers.push_back(Chain);
8297 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8298 Opers.push_back(getValue(FPI.getArgOperand(I)));
8299
8300 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
8301 assert(Result.getNode()->getNumValues() == 2);
8302
8303 // Push node to the appropriate list so that future instructions can be
8304 // chained up correctly.
8305 SDValue OutChain = Result.getValue(1);
8306 switch (EB) {
8308 // The only reason why ebIgnore nodes still need to be chained is that
8309 // they might depend on the current rounding mode, and therefore must
8310 // not be moved across instruction that may change that mode.
8311 [[fallthrough]];
8313 // These must not be moved across calls or instructions that may change
8314 // floating-point exception masks.
8315 PendingConstrainedFP.push_back(OutChain);
8316 break;
8318 // These must not be moved across calls or instructions that may change
8319 // floating-point exception masks or read floating-point exception flags.
8320 // In addition, they cannot be optimized out even if unused.
8321 PendingConstrainedFPStrict.push_back(OutChain);
8322 break;
8323 }
8324 };
8325
8326 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8327 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
8328 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
8330
8331 SDNodeFlags Flags;
8333 Flags.setNoFPExcept(true);
8334
8335 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
8336 Flags.copyFMF(*FPOp);
8337
8338 unsigned Opcode;
8339 switch (FPI.getIntrinsicID()) {
8340 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8341#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8342 case Intrinsic::INTRINSIC: \
8343 Opcode = ISD::STRICT_##DAGN; \
8344 break;
8345#include "llvm/IR/ConstrainedOps.def"
8346 case Intrinsic::experimental_constrained_fmuladd: {
8347 Opcode = ISD::STRICT_FMA;
8348 // Break fmuladd into fmul and fadd.
8349 if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
8350 !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
8351 Opers.pop_back();
8352 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8353 pushOutChain(Mul, EB);
8354 Opcode = ISD::STRICT_FADD;
8355 Opers.clear();
8356 Opers.push_back(Mul.getValue(1));
8357 Opers.push_back(Mul.getValue(0));
8358 Opers.push_back(getValue(FPI.getArgOperand(2)));
8359 }
8360 break;
8361 }
8362 }
8363
8364 // A few strict DAG nodes carry additional operands that are not
8365 // set up by the default code above.
8366 switch (Opcode) {
8367 default: break;
8369 Opers.push_back(
8370 DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
8371 break;
8372 case ISD::STRICT_FSETCC:
8373 case ISD::STRICT_FSETCCS: {
8375 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8376 if (TM.Options.NoNaNsFPMath)
8377 Condition = getFCmpCodeWithoutNaN(Condition);
8378 Opers.push_back(DAG.getCondCode(Condition));
8379 break;
8380 }
8381 }
8382
8383 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8384 pushOutChain(Result, EB);
8385
8386 SDValue FPResult = Result.getValue(0);
8387 setValue(&FPI, FPResult);
8388}
8389
8390static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8391 std::optional<unsigned> ResOPC;
8392 switch (VPIntrin.getIntrinsicID()) {
8393 case Intrinsic::vp_ctlz: {
8394 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8395 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8396 break;
8397 }
8398 case Intrinsic::vp_cttz: {
8399 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8400 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8401 break;
8402 }
8403 case Intrinsic::vp_cttz_elts: {
8404 bool IsZeroPoison = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8405 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_UNDEF : ISD::VP_CTTZ_ELTS;
8406 break;
8407 }
8408#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8409 case Intrinsic::VPID: \
8410 ResOPC = ISD::VPSD; \
8411 break;
8412#include "llvm/IR/VPIntrinsics.def"
8413 }
8414
8415 if (!ResOPC)
8417 "Inconsistency: no SDNode available for this VPIntrinsic!");
8418
8419 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8420 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8421 if (VPIntrin.getFastMathFlags().allowReassoc())
8422 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8423 : ISD::VP_REDUCE_FMUL;
8424 }
8425
8426 return *ResOPC;
8427}
8428
8429void SelectionDAGBuilder::visitVPLoad(
8430 const VPIntrinsic &VPIntrin, EVT VT,
8431 const SmallVectorImpl<SDValue> &OpValues) {
8432 SDLoc DL = getCurSDLoc();
8433 Value *PtrOperand = VPIntrin.getArgOperand(0);
8434 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8435 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8436 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8437 SDValue LD;
8438 // Do not serialize variable-length loads of constant memory with
8439 // anything.
8440 if (!Alignment)
8441 Alignment = DAG.getEVTAlign(VT);
8442 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8443 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
8444 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8445 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8446 MachineMemOperand::Flags MMOFlags =
8447 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8448 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8449 MachinePointerInfo(PtrOperand), MMOFlags,
8450 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8451 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
8452 MMO, false /*IsExpanding */);
8453 if (AddToChain)
8454 PendingLoads.push_back(LD.getValue(1));
8455 setValue(&VPIntrin, LD);
8456}
8457
8458void SelectionDAGBuilder::visitVPLoadFF(
8459 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,
8460 const SmallVectorImpl<SDValue> &OpValues) {
8461 assert(OpValues.size() == 3 && "Unexpected number of operands");
8462 SDLoc DL = getCurSDLoc();
8463 Value *PtrOperand = VPIntrin.getArgOperand(0);
8464 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8465 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8466 const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
8467 SDValue LD;
8468 // Do not serialize variable-length loads of constant memory with
8469 // anything.
8470 if (!Alignment)
8471 Alignment = DAG.getEVTAlign(VT);
8472 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8473 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
8474 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8475 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8476 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
8477 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8478 LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
8479 MMO);
8480 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));
8481 if (AddToChain)
8482 PendingLoads.push_back(LD.getValue(2));
8483 setValue(&VPIntrin, DAG.getMergeValues({LD.getValue(0), Trunc}, DL));
8484}
8485
8486void SelectionDAGBuilder::visitVPGather(
8487 const VPIntrinsic &VPIntrin, EVT VT,
8488 const SmallVectorImpl<SDValue> &OpValues) {
8489 SDLoc DL = getCurSDLoc();
8490 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8491 Value *PtrOperand = VPIntrin.getArgOperand(0);
8492 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8493 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8494 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8495 SDValue LD;
8496 if (!Alignment)
8497 Alignment = DAG.getEVTAlign(VT.getScalarType());
8498 unsigned AS =
8499 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8500 MachineMemOperand::Flags MMOFlags =
8501 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8502 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8503 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
8504 *Alignment, AAInfo, Ranges);
8505 SDValue Base, Index, Scale;
8506 bool UniformBase =
8507 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
8508 VT.getScalarStoreSize());
8509 if (!UniformBase) {
8510 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
8511 Index = getValue(PtrOperand);
8512 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
8513 }
8514 EVT IdxVT = Index.getValueType();
8515 EVT EltTy = IdxVT.getVectorElementType();
8516 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8517 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8518 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8519 }
8520 LD = DAG.getGatherVP(
8521 DAG.getVTList(VT, MVT::Other), VT, DL,
8522 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8524 PendingLoads.push_back(LD.getValue(1));
8525 setValue(&VPIntrin, LD);
8526}
8527
8528void SelectionDAGBuilder::visitVPStore(
8529 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8530 SDLoc DL = getCurSDLoc();
8531 Value *PtrOperand = VPIntrin.getArgOperand(1);
8532 EVT VT = OpValues[0].getValueType();
8533 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8534 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8535 SDValue ST;
8536 if (!Alignment)
8537 Alignment = DAG.getEVTAlign(VT);
8538 SDValue Ptr = OpValues[1];
8539 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
8540 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8541 MachineMemOperand::Flags MMOFlags =
8542 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8543 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8544 MachinePointerInfo(PtrOperand), MMOFlags,
8545 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8546 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
8547 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
8548 /* IsTruncating */ false, /*IsCompressing*/ false);
8549 DAG.setRoot(ST);
8550 setValue(&VPIntrin, ST);
8551}
8552
8553void SelectionDAGBuilder::visitVPScatter(
8554 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8555 SDLoc DL = getCurSDLoc();
8556 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8557 Value *PtrOperand = VPIntrin.getArgOperand(1);
8558 EVT VT = OpValues[0].getValueType();
8559 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8560 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8561 SDValue ST;
8562 if (!Alignment)
8563 Alignment = DAG.getEVTAlign(VT.getScalarType());
8564 unsigned AS =
8565 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8566 MachineMemOperand::Flags MMOFlags =
8567 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8568 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8569 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
8570 *Alignment, AAInfo);
8571 SDValue Base, Index, Scale;
8572 bool UniformBase =
8573 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
8574 VT.getScalarStoreSize());
8575 if (!UniformBase) {
8576 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
8577 Index = getValue(PtrOperand);
8578 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
8579 }
8580 EVT IdxVT = Index.getValueType();
8581 EVT EltTy = IdxVT.getVectorElementType();
8582 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8583 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8584 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8585 }
8586 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
8587 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8588 OpValues[2], OpValues[3]},
8589 MMO, ISD::SIGNED_SCALED);
8590 DAG.setRoot(ST);
8591 setValue(&VPIntrin, ST);
8592}
8593
8594void SelectionDAGBuilder::visitVPStridedLoad(
8595 const VPIntrinsic &VPIntrin, EVT VT,
8596 const SmallVectorImpl<SDValue> &OpValues) {
8597 SDLoc DL = getCurSDLoc();
8598 Value *PtrOperand = VPIntrin.getArgOperand(0);
8599 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8600 if (!Alignment)
8601 Alignment = DAG.getEVTAlign(VT.getScalarType());
8602 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8603 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8604 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8605 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
8606 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8607 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8608 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8609 MachineMemOperand::Flags MMOFlags =
8610 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8611 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8612 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
8613 *Alignment, AAInfo, Ranges);
8614
8615 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
8616 OpValues[2], OpValues[3], MMO,
8617 false /*IsExpanding*/);
8618
8619 if (AddToChain)
8620 PendingLoads.push_back(LD.getValue(1));
8621 setValue(&VPIntrin, LD);
8622}
8623
8624void SelectionDAGBuilder::visitVPStridedStore(
8625 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8626 SDLoc DL = getCurSDLoc();
8627 Value *PtrOperand = VPIntrin.getArgOperand(1);
8628 EVT VT = OpValues[0].getValueType();
8629 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8630 if (!Alignment)
8631 Alignment = DAG.getEVTAlign(VT.getScalarType());
8632 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8633 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8634 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8635 MachineMemOperand::Flags MMOFlags =
8636 TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
8637 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8638 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
8639 *Alignment, AAInfo);
8640
8641 SDValue ST = DAG.getStridedStoreVP(
8642 getMemoryRoot(), DL, OpValues[0], OpValues[1],
8643 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
8644 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
8645 /*IsCompressing*/ false);
8646
8647 DAG.setRoot(ST);
8648 setValue(&VPIntrin, ST);
8649}
8650
8651void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8652 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8653 SDLoc DL = getCurSDLoc();
8654
8655 ISD::CondCode Condition;
8657 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
8658 if (IsFP) {
8659 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
8660 // flags, but calls that don't return floating-point types can't be
8661 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
8662 Condition = getFCmpCondCode(CondCode);
8663 if (TM.Options.NoNaNsFPMath)
8664 Condition = getFCmpCodeWithoutNaN(Condition);
8665 } else {
8666 Condition = getICmpCondCode(CondCode);
8667 }
8668
8669 SDValue Op1 = getValue(VPIntrin.getOperand(0));
8670 SDValue Op2 = getValue(VPIntrin.getOperand(1));
8671 // #2 is the condition code
8672 SDValue MaskOp = getValue(VPIntrin.getOperand(3));
8673 SDValue EVL = getValue(VPIntrin.getOperand(4));
8674 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8675 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8676 "Unexpected target EVL type");
8677 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
8678
8679 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
8680 VPIntrin.getType());
8681 setValue(&VPIntrin,
8682 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
8683}
8684
8685void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8686 const VPIntrinsic &VPIntrin) {
8687 SDLoc DL = getCurSDLoc();
8688 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8689
8690 auto IID = VPIntrin.getIntrinsicID();
8691
8692 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
8693 return visitVPCmp(*CmpI);
8694
8695 SmallVector<EVT, 4> ValueVTs;
8696 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8697 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
8698 SDVTList VTs = DAG.getVTList(ValueVTs);
8699
8700 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
8701
8702 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8703 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8704 "Unexpected target EVL type");
8705
8706 // Request operands.
8707 SmallVector<SDValue, 7> OpValues;
8708 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8709 auto Op = getValue(VPIntrin.getArgOperand(I));
8710 if (I == EVLParamPos)
8711 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
8712 OpValues.push_back(Op);
8713 }
8714
8715 switch (Opcode) {
8716 default: {
8717 SDNodeFlags SDFlags;
8718 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8719 SDFlags.copyFMF(*FPMO);
8720 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
8721 setValue(&VPIntrin, Result);
8722 break;
8723 }
8724 case ISD::VP_LOAD:
8725 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
8726 break;
8727 case ISD::VP_LOAD_FF:
8728 visitVPLoadFF(VPIntrin, ValueVTs[0], ValueVTs[1], OpValues);
8729 break;
8730 case ISD::VP_GATHER:
8731 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
8732 break;
8733 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8734 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
8735 break;
8736 case ISD::VP_STORE:
8737 visitVPStore(VPIntrin, OpValues);
8738 break;
8739 case ISD::VP_SCATTER:
8740 visitVPScatter(VPIntrin, OpValues);
8741 break;
8742 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8743 visitVPStridedStore(VPIntrin, OpValues);
8744 break;
8745 case ISD::VP_FMULADD: {
8746 assert(OpValues.size() == 5 && "Unexpected number of operands");
8747 SDNodeFlags SDFlags;
8748 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8749 SDFlags.copyFMF(*FPMO);
8750 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
8751 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), ValueVTs[0])) {
8752 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
8753 } else {
8754 SDValue Mul = DAG.getNode(
8755 ISD::VP_FMUL, DL, VTs,
8756 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
8757 SDValue Add =
8758 DAG.getNode(ISD::VP_FADD, DL, VTs,
8759 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
8760 setValue(&VPIntrin, Add);
8761 }
8762 break;
8763 }
8764 case ISD::VP_IS_FPCLASS: {
8765 const DataLayout DLayout = DAG.getDataLayout();
8766 EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType());
8767 auto Constant = OpValues[1]->getAsZExtVal();
8768 SDValue Check = DAG.getTargetConstant(Constant, DL, MVT::i32);
8769 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT,
8770 {OpValues[0], Check, OpValues[2], OpValues[3]});
8771 setValue(&VPIntrin, V);
8772 return;
8773 }
8774 case ISD::VP_INTTOPTR: {
8775 SDValue N = OpValues[0];
8776 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
8777 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
8778 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8779 OpValues[2]);
8780 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8781 OpValues[2]);
8782 setValue(&VPIntrin, N);
8783 break;
8784 }
8785 case ISD::VP_PTRTOINT: {
8786 SDValue N = OpValues[0];
8787 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
8788 VPIntrin.getType());
8789 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
8790 VPIntrin.getOperand(0)->getType());
8791 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8792 OpValues[2]);
8793 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8794 OpValues[2]);
8795 setValue(&VPIntrin, N);
8796 break;
8797 }
8798 case ISD::VP_ABS:
8799 case ISD::VP_CTLZ:
8800 case ISD::VP_CTLZ_ZERO_UNDEF:
8801 case ISD::VP_CTTZ:
8802 case ISD::VP_CTTZ_ZERO_UNDEF:
8803 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
8804 case ISD::VP_CTTZ_ELTS: {
8805 SDValue Result =
8806 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
8807 setValue(&VPIntrin, Result);
8808 break;
8809 }
8810 }
8811}
8812
8813SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
8814 const BasicBlock *EHPadBB,
8815 MCSymbol *&BeginLabel) {
8816 MachineFunction &MF = DAG.getMachineFunction();
8817
8818 // Insert a label before the invoke call to mark the try range. This can be
8819 // used to detect deletion of the invoke via the MachineModuleInfo.
8820 BeginLabel = MF.getContext().createTempSymbol();
8821
8822 // For SjLj, keep track of which landing pads go with which invokes
8823 // so as to maintain the ordering of pads in the LSDA.
8824 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
8825 if (CallSiteIndex) {
8826 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
8827 LPadToCallSiteMap[FuncInfo.getMBB(EHPadBB)].push_back(CallSiteIndex);
8828
8829 // Now that the call site is handled, stop tracking it.
8830 FuncInfo.setCurrentCallSite(0);
8831 }
8832
8833 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
8834}
8835
8836SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
8837 const BasicBlock *EHPadBB,
8838 MCSymbol *BeginLabel) {
8839 assert(BeginLabel && "BeginLabel should've been set");
8840
8841 MachineFunction &MF = DAG.getMachineFunction();
8842
8843 // Insert a label at the end of the invoke call to mark the try range. This
8844 // can be used to detect deletion of the invoke via the MachineModuleInfo.
8845 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
8846 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
8847
8848 // Inform MachineModuleInfo of range.
8849 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
8850 // There is a platform (e.g. wasm) that uses funclet style IR but does not
8851 // actually use outlined funclets and their LSDA info style.
8852 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
8853 assert(II && "II should've been set");
8854 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
8855 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
8856 } else if (!isScopedEHPersonality(Pers)) {
8857 assert(EHPadBB);
8858 MF.addInvoke(FuncInfo.getMBB(EHPadBB), BeginLabel, EndLabel);
8859 }
8860
8861 return Chain;
8862}
8863
8864std::pair<SDValue, SDValue>
8866 const BasicBlock *EHPadBB) {
8867 MCSymbol *BeginLabel = nullptr;
8868
8869 if (EHPadBB) {
8870 // Both PendingLoads and PendingExports must be flushed here;
8871 // this call might not return.
8872 (void)getRoot();
8873 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
8874 CLI.setChain(getRoot());
8875 }
8876
8877 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8878 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8879
8880 assert((CLI.IsTailCall || Result.second.getNode()) &&
8881 "Non-null chain expected with non-tail call!");
8882 assert((Result.second.getNode() || !Result.first.getNode()) &&
8883 "Null value expected with tail call!");
8884
8885 if (!Result.second.getNode()) {
8886 // As a special case, a null chain means that a tail call has been emitted
8887 // and the DAG root is already updated.
8888 HasTailCall = true;
8889
8890 // Since there's no actual continuation from this block, nothing can be
8891 // relying on us setting vregs for them.
8892 PendingExports.clear();
8893 } else {
8894 DAG.setRoot(Result.second);
8895 }
8896
8897 if (EHPadBB) {
8898 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
8899 BeginLabel));
8900 Result.second = getRoot();
8901 }
8902
8903 return Result;
8904}
8905
8907 bool isMustTailCall = CB.isMustTailCall();
8908
8909 // Avoid emitting tail calls in functions with the disable-tail-calls
8910 // attribute.
8911 const Function *Caller = CB.getParent()->getParent();
8912 if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
8913 "true" &&
8914 !isMustTailCall)
8915 return false;
8916
8917 // We can't tail call inside a function with a swifterror argument. Lowering
8918 // does not support this yet. It would have to move into the swifterror
8919 // register before the call.
8920 if (DAG.getTargetLoweringInfo().supportSwiftError() &&
8921 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8922 return false;
8923
8924 // Check if target-independent constraints permit a tail call here.
8925 // Target-dependent constraints are checked within TLI->LowerCallTo.
8926 return isInTailCallPosition(CB, DAG.getTarget());
8927}
8928
8930 bool isTailCall, bool isMustTailCall,
8931 const BasicBlock *EHPadBB,
8932 const TargetLowering::PtrAuthInfo *PAI) {
8933 auto &DL = DAG.getDataLayout();
8934 FunctionType *FTy = CB.getFunctionType();
8935 Type *RetTy = CB.getType();
8936
8938 Args.reserve(CB.arg_size());
8939
8940 const Value *SwiftErrorVal = nullptr;
8941 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8942
8943 if (isTailCall)
8944 isTailCall = canTailCall(CB);
8945
8946 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
8947 const Value *V = *I;
8948
8949 // Skip empty types
8950 if (V->getType()->isEmptyTy())
8951 continue;
8952
8953 SDValue ArgNode = getValue(V);
8954 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
8955 Entry.setAttributes(&CB, I - CB.arg_begin());
8956
8957 // Use swifterror virtual register as input to the call.
8958 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
8959 SwiftErrorVal = V;
8960 // We find the virtual register for the actual swifterror argument.
8961 // Instead of using the Value, we use the virtual register instead.
8962 Entry.Node =
8963 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
8964 EVT(TLI.getPointerTy(DL)));
8965 }
8966
8967 Args.push_back(Entry);
8968
8969 // If we have an explicit sret argument that is an Instruction, (i.e., it
8970 // might point to function-local memory), we can't meaningfully tail-call.
8971 if (Entry.IsSRet && isa<Instruction>(V))
8972 isTailCall = false;
8973 }
8974
8975 // If call site has a cfguardtarget operand bundle, create and add an
8976 // additional ArgListEntry.
8977 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
8978 Value *V = Bundle->Inputs[0];
8980 Entry.IsCFGuardTarget = true;
8981 Args.push_back(Entry);
8982 }
8983
8984 // Disable tail calls if there is an swifterror argument. Targets have not
8985 // been updated to support tail calls.
8986 if (TLI.supportSwiftError() && SwiftErrorVal)
8987 isTailCall = false;
8988
8989 ConstantInt *CFIType = nullptr;
8990 if (CB.isIndirectCall()) {
8991 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
8992 if (!TLI.supportKCFIBundles())
8994 "Target doesn't support calls with kcfi operand bundles.");
8995 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8996 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8997 }
8998 }
8999
9000 SDValue ConvControlToken;
9001 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
9002 auto *Token = Bundle->Inputs[0].get();
9003 ConvControlToken = getValue(Token);
9004 }
9005
9008 .setChain(getRoot())
9009 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
9010 .setTailCall(isTailCall)
9014 .setCFIType(CFIType)
9015 .setConvergenceControlToken(ConvControlToken);
9016
9017 // Set the pointer authentication info if we have it.
9018 if (PAI) {
9019 if (!TLI.supportPtrAuthBundles())
9021 "This target doesn't support calls with ptrauth operand bundles.");
9022 CLI.setPtrAuth(*PAI);
9023 }
9024
9025 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9026
9027 if (Result.first.getNode()) {
9028 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
9029 setValue(&CB, Result.first);
9030 }
9031
9032 // The last element of CLI.InVals has the SDValue for swifterror return.
9033 // Here we copy it to a virtual register and update SwiftErrorMap for
9034 // book-keeping.
9035 if (SwiftErrorVal && TLI.supportSwiftError()) {
9036 // Get the last element of InVals.
9037 SDValue Src = CLI.InVals.back();
9038 Register VReg =
9039 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
9040 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
9041 DAG.setRoot(CopyNode);
9042 }
9043}
9044
9045static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
9046 SelectionDAGBuilder &Builder) {
9047 // Check to see if this load can be trivially constant folded, e.g. if the
9048 // input is from a string literal.
9049 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
9050 // Cast pointer to the type we really want to load.
9051 Type *LoadTy =
9052 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
9053 if (LoadVT.isVector())
9054 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
9055 if (const Constant *LoadCst =
9056 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
9057 LoadTy, Builder.DAG.getDataLayout()))
9058 return Builder.getValue(LoadCst);
9059 }
9060
9061 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9062 // still constant memory, the input chain can be the entry node.
9063 SDValue Root;
9064 bool ConstantMemory = false;
9065
9066 // Do not serialize (non-volatile) loads of constant memory with anything.
9067 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) {
9068 Root = Builder.DAG.getEntryNode();
9069 ConstantMemory = true;
9070 } else {
9071 // Do not serialize non-volatile loads against each other.
9072 Root = Builder.DAG.getRoot();
9073 }
9074
9075 SDValue Ptr = Builder.getValue(PtrVal);
9076 SDValue LoadVal =
9077 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
9078 MachinePointerInfo(PtrVal), Align(1));
9079
9080 if (!ConstantMemory)
9081 Builder.PendingLoads.push_back(LoadVal.getValue(1));
9082 return LoadVal;
9083}
9084
9085/// Record the value for an instruction that produces an integer result,
9086/// converting the type where necessary.
9087void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9088 SDValue Value,
9089 bool IsSigned) {
9090 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
9091 I.getType(), true);
9092 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
9093 setValue(&I, Value);
9094}
9095
9096/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9097/// true and lower it. Otherwise return false, and it will be lowered like a
9098/// normal call.
9099/// The caller already checked that \p I calls the appropriate LibFunc with a
9100/// correct prototype.
9101bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9102 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
9103 const Value *Size = I.getArgOperand(2);
9104 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
9105 if (CSize && CSize->getZExtValue() == 0) {
9106 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
9107 I.getType(), true);
9108 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
9109 return true;
9110 }
9111
9112 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9113 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9114 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
9115 getValue(Size), &I);
9116 if (Res.first.getNode()) {
9117 processIntegerCallValue(I, Res.first, true);
9118 PendingLoads.push_back(Res.second);
9119 return true;
9120 }
9121
9122 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9123 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9124 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
9125 return false;
9126
9127 // If the target has a fast compare for the given size, it will return a
9128 // preferred load type for that size. Require that the load VT is legal and
9129 // that the target supports unaligned loads of that type. Otherwise, return
9130 // INVALID.
9131 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9132 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9133 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9134 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9135 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9136 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9137 // TODO: Check alignment of src and dest ptrs.
9138 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9139 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9140 if (!TLI.isTypeLegal(LVT) ||
9141 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
9142 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
9144 }
9145
9146 return LVT;
9147 };
9148
9149 // This turns into unaligned loads. We only do this if the target natively
9150 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9151 // we'll only produce a small number of byte loads.
9152 MVT LoadVT;
9153 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9154 switch (NumBitsToCompare) {
9155 default:
9156 return false;
9157 case 16:
9158 LoadVT = MVT::i16;
9159 break;
9160 case 32:
9161 LoadVT = MVT::i32;
9162 break;
9163 case 64:
9164 case 128:
9165 case 256:
9166 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9167 break;
9168 }
9169
9170 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9171 return false;
9172
9173 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
9174 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
9175
9176 // Bitcast to a wide integer type if the loads are vectors.
9177 if (LoadVT.isVector()) {
9178 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
9179 LoadL = DAG.getBitcast(CmpVT, LoadL);
9180 LoadR = DAG.getBitcast(CmpVT, LoadR);
9181 }
9182
9183 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
9184 processIntegerCallValue(I, Cmp, false);
9185 return true;
9186}
9187
9188/// See if we can lower a memchr call into an optimized form. If so, return
9189/// true and lower it. Otherwise return false, and it will be lowered like a
9190/// normal call.
9191/// The caller already checked that \p I calls the appropriate LibFunc with a
9192/// correct prototype.
9193bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9194 const Value *Src = I.getArgOperand(0);
9195 const Value *Char = I.getArgOperand(1);
9196 const Value *Length = I.getArgOperand(2);
9197
9198 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9199 std::pair<SDValue, SDValue> Res =
9200 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
9201 getValue(Src), getValue(Char), getValue(Length),
9202 MachinePointerInfo(Src));
9203 if (Res.first.getNode()) {
9204 setValue(&I, Res.first);
9205 PendingLoads.push_back(Res.second);
9206 return true;
9207 }
9208
9209 return false;
9210}
9211
9212/// See if we can lower a mempcpy call into an optimized form. If so, return
9213/// true and lower it. Otherwise return false, and it will be lowered like a
9214/// normal call.
9215/// The caller already checked that \p I calls the appropriate LibFunc with a
9216/// correct prototype.
9217bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9218 SDValue Dst = getValue(I.getArgOperand(0));
9219 SDValue Src = getValue(I.getArgOperand(1));
9220 SDValue Size = getValue(I.getArgOperand(2));
9221
9222 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
9223 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
9224 // DAG::getMemcpy needs Alignment to be defined.
9225 Align Alignment = std::min(DstAlign, SrcAlign);
9226
9227 SDLoc sdl = getCurSDLoc();
9228
9229 // In the mempcpy context we need to pass in a false value for isTailCall
9230 // because the return pointer needs to be adjusted by the size of
9231 // the copied memory.
9232 SDValue Root = getMemoryRoot();
9233 SDValue MC = DAG.getMemcpy(
9234 Root, sdl, Dst, Src, Size, Alignment, false, false, /*CI=*/nullptr,
9235 std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
9236 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
9237 assert(MC.getNode() != nullptr &&
9238 "** memcpy should not be lowered as TailCall in mempcpy context **");
9239 DAG.setRoot(MC);
9240
9241 // Check if Size needs to be truncated or extended.
9242 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
9243
9244 // Adjust return pointer to point just past the last dst byte.
9245 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Dst, Size, sdl);
9246 setValue(&I, DstPlusSize);
9247 return true;
9248}
9249
9250/// See if we can lower a strcpy call into an optimized form. If so, return
9251/// true and lower it, otherwise return false and it will be lowered like a
9252/// normal call.
9253/// The caller already checked that \p I calls the appropriate LibFunc with a
9254/// correct prototype.
9255bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9256 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9257
9258 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9259 std::pair<SDValue, SDValue> Res =
9261 getValue(Arg0), getValue(Arg1),
9262 MachinePointerInfo(Arg0),
9263 MachinePointerInfo(Arg1), isStpcpy);
9264 if (Res.first.getNode()) {
9265 setValue(&I, Res.first);
9266 DAG.setRoot(Res.second);
9267 return true;
9268 }
9269
9270 return false;
9271}
9272
9273/// See if we can lower a strcmp call into an optimized form. If so, return
9274/// true and lower it, otherwise return false and it will be lowered like a
9275/// normal call.
9276/// The caller already checked that \p I calls the appropriate LibFunc with a
9277/// correct prototype.
9278bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9279 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9280
9281 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9282 std::pair<SDValue, SDValue> Res =
9283 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
9284 getValue(Arg0), getValue(Arg1),
9285 MachinePointerInfo(Arg0),
9286 MachinePointerInfo(Arg1));
9287 if (Res.first.getNode()) {
9288 processIntegerCallValue(I, Res.first, true);
9289 PendingLoads.push_back(Res.second);
9290 return true;
9291 }
9292
9293 return false;
9294}
9295
9296/// See if we can lower a strlen call into an optimized form. If so, return
9297/// true and lower it, otherwise return false and it will be lowered like a
9298/// normal call.
9299/// The caller already checked that \p I calls the appropriate LibFunc with a
9300/// correct prototype.
9301bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9302 const Value *Arg0 = I.getArgOperand(0);
9303
9304 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9305 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(
9306 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), &I);
9307 if (Res.first.getNode()) {
9308 processIntegerCallValue(I, Res.first, false);
9309 PendingLoads.push_back(Res.second);
9310 return true;
9311 }
9312
9313 return false;
9314}
9315
9316/// See if we can lower a strnlen call into an optimized form. If so, return
9317/// true and lower it, otherwise return false and it will be lowered like a
9318/// normal call.
9319/// The caller already checked that \p I calls the appropriate LibFunc with a
9320/// correct prototype.
9321bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9322 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9323
9324 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9325 std::pair<SDValue, SDValue> Res =
9326 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
9327 getValue(Arg0), getValue(Arg1),
9328 MachinePointerInfo(Arg0));
9329 if (Res.first.getNode()) {
9330 processIntegerCallValue(I, Res.first, false);
9331 PendingLoads.push_back(Res.second);
9332 return true;
9333 }
9334
9335 return false;
9336}
9337
9338/// See if we can lower a unary floating-point operation into an SDNode with
9339/// the specified Opcode. If so, return true and lower it, otherwise return
9340/// false and it will be lowered like a normal call.
9341/// The caller already checked that \p I calls the appropriate LibFunc with a
9342/// correct prototype.
9343bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9344 unsigned Opcode) {
9345 // We already checked this call's prototype; verify it doesn't modify errno.
9346 if (!I.onlyReadsMemory())
9347 return false;
9348
9349 SDNodeFlags Flags;
9350 Flags.copyFMF(cast<FPMathOperator>(I));
9351
9352 SDValue Tmp = getValue(I.getArgOperand(0));
9353 setValue(&I,
9354 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
9355 return true;
9356}
9357
9358/// See if we can lower a binary floating-point operation into an SDNode with
9359/// the specified Opcode. If so, return true and lower it. Otherwise return
9360/// false, and it will be lowered like a normal call.
9361/// The caller already checked that \p I calls the appropriate LibFunc with a
9362/// correct prototype.
9363bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9364 unsigned Opcode) {
9365 // We already checked this call's prototype; verify it doesn't modify errno.
9366 if (!I.onlyReadsMemory())
9367 return false;
9368
9369 SDNodeFlags Flags;
9370 Flags.copyFMF(cast<FPMathOperator>(I));
9371
9372 SDValue Tmp0 = getValue(I.getArgOperand(0));
9373 SDValue Tmp1 = getValue(I.getArgOperand(1));
9374 EVT VT = Tmp0.getValueType();
9375 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
9376 return true;
9377}
9378
9379void SelectionDAGBuilder::visitCall(const CallInst &I) {
9380 // Handle inline assembly differently.
9381 if (I.isInlineAsm()) {
9382 visitInlineAsm(I);
9383 return;
9384 }
9385
9387
9388 if (Function *F = I.getCalledFunction()) {
9389 if (F->isDeclaration()) {
9390 // Is this an LLVM intrinsic?
9391 if (unsigned IID = F->getIntrinsicID()) {
9392 visitIntrinsicCall(I, IID);
9393 return;
9394 }
9395 }
9396
9397 // Check for well-known libc/libm calls. If the function is internal, it
9398 // can't be a library call. Don't do the check if marked as nobuiltin for
9399 // some reason or the call site requires strict floating point semantics.
9400 LibFunc Func;
9401 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
9402 F->hasName() && LibInfo->getLibFunc(*F, Func) &&
9403 LibInfo->hasOptimizedCodeGen(Func)) {
9404 switch (Func) {
9405 default: break;
9406 case LibFunc_bcmp:
9407 if (visitMemCmpBCmpCall(I))
9408 return;
9409 break;
9410 case LibFunc_copysign:
9411 case LibFunc_copysignf:
9412 case LibFunc_copysignl:
9413 // We already checked this call's prototype; verify it doesn't modify
9414 // errno.
9415 if (I.onlyReadsMemory()) {
9416 SDValue LHS = getValue(I.getArgOperand(0));
9417 SDValue RHS = getValue(I.getArgOperand(1));
9419 LHS.getValueType(), LHS, RHS));
9420 return;
9421 }
9422 break;
9423 case LibFunc_fabs:
9424 case LibFunc_fabsf:
9425 case LibFunc_fabsl:
9426 if (visitUnaryFloatCall(I, ISD::FABS))
9427 return;
9428 break;
9429 case LibFunc_fmin:
9430 case LibFunc_fminf:
9431 case LibFunc_fminl:
9432 if (visitBinaryFloatCall(I, ISD::FMINNUM))
9433 return;
9434 break;
9435 case LibFunc_fmax:
9436 case LibFunc_fmaxf:
9437 case LibFunc_fmaxl:
9438 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
9439 return;
9440 break;
9441 case LibFunc_fminimum_num:
9442 case LibFunc_fminimum_numf:
9443 case LibFunc_fminimum_numl:
9444 if (visitBinaryFloatCall(I, ISD::FMINIMUMNUM))
9445 return;
9446 break;
9447 case LibFunc_fmaximum_num:
9448 case LibFunc_fmaximum_numf:
9449 case LibFunc_fmaximum_numl:
9450 if (visitBinaryFloatCall(I, ISD::FMAXIMUMNUM))
9451 return;
9452 break;
9453 case LibFunc_sin:
9454 case LibFunc_sinf:
9455 case LibFunc_sinl:
9456 if (visitUnaryFloatCall(I, ISD::FSIN))
9457 return;
9458 break;
9459 case LibFunc_cos:
9460 case LibFunc_cosf:
9461 case LibFunc_cosl:
9462 if (visitUnaryFloatCall(I, ISD::FCOS))
9463 return;
9464 break;
9465 case LibFunc_tan:
9466 case LibFunc_tanf:
9467 case LibFunc_tanl:
9468 if (visitUnaryFloatCall(I, ISD::FTAN))
9469 return;
9470 break;
9471 case LibFunc_asin:
9472 case LibFunc_asinf:
9473 case LibFunc_asinl:
9474 if (visitUnaryFloatCall(I, ISD::FASIN))
9475 return;
9476 break;
9477 case LibFunc_acos:
9478 case LibFunc_acosf:
9479 case LibFunc_acosl:
9480 if (visitUnaryFloatCall(I, ISD::FACOS))
9481 return;
9482 break;
9483 case LibFunc_atan:
9484 case LibFunc_atanf:
9485 case LibFunc_atanl:
9486 if (visitUnaryFloatCall(I, ISD::FATAN))
9487 return;
9488 break;
9489 case LibFunc_atan2:
9490 case LibFunc_atan2f:
9491 case LibFunc_atan2l:
9492 if (visitBinaryFloatCall(I, ISD::FATAN2))
9493 return;
9494 break;
9495 case LibFunc_sinh:
9496 case LibFunc_sinhf:
9497 case LibFunc_sinhl:
9498 if (visitUnaryFloatCall(I, ISD::FSINH))
9499 return;
9500 break;
9501 case LibFunc_cosh:
9502 case LibFunc_coshf:
9503 case LibFunc_coshl:
9504 if (visitUnaryFloatCall(I, ISD::FCOSH))
9505 return;
9506 break;
9507 case LibFunc_tanh:
9508 case LibFunc_tanhf:
9509 case LibFunc_tanhl:
9510 if (visitUnaryFloatCall(I, ISD::FTANH))
9511 return;
9512 break;
9513 case LibFunc_sqrt:
9514 case LibFunc_sqrtf:
9515 case LibFunc_sqrtl:
9516 case LibFunc_sqrt_finite:
9517 case LibFunc_sqrtf_finite:
9518 case LibFunc_sqrtl_finite:
9519 if (visitUnaryFloatCall(I, ISD::FSQRT))
9520 return;
9521 break;
9522 case LibFunc_floor:
9523 case LibFunc_floorf:
9524 case LibFunc_floorl:
9525 if (visitUnaryFloatCall(I, ISD::FFLOOR))
9526 return;
9527 break;
9528 case LibFunc_nearbyint:
9529 case LibFunc_nearbyintf:
9530 case LibFunc_nearbyintl:
9531 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
9532 return;
9533 break;
9534 case LibFunc_ceil:
9535 case LibFunc_ceilf:
9536 case LibFunc_ceill:
9537 if (visitUnaryFloatCall(I, ISD::FCEIL))
9538 return;
9539 break;
9540 case LibFunc_rint:
9541 case LibFunc_rintf:
9542 case LibFunc_rintl:
9543 if (visitUnaryFloatCall(I, ISD::FRINT))
9544 return;
9545 break;
9546 case LibFunc_round:
9547 case LibFunc_roundf:
9548 case LibFunc_roundl:
9549 if (visitUnaryFloatCall(I, ISD::FROUND))
9550 return;
9551 break;
9552 case LibFunc_trunc:
9553 case LibFunc_truncf:
9554 case LibFunc_truncl:
9555 if (visitUnaryFloatCall(I, ISD::FTRUNC))
9556 return;
9557 break;
9558 case LibFunc_log2:
9559 case LibFunc_log2f:
9560 case LibFunc_log2l:
9561 if (visitUnaryFloatCall(I, ISD::FLOG2))
9562 return;
9563 break;
9564 case LibFunc_exp2:
9565 case LibFunc_exp2f:
9566 case LibFunc_exp2l:
9567 if (visitUnaryFloatCall(I, ISD::FEXP2))
9568 return;
9569 break;
9570 case LibFunc_exp10:
9571 case LibFunc_exp10f:
9572 case LibFunc_exp10l:
9573 if (visitUnaryFloatCall(I, ISD::FEXP10))
9574 return;
9575 break;
9576 case LibFunc_ldexp:
9577 case LibFunc_ldexpf:
9578 case LibFunc_ldexpl:
9579 if (visitBinaryFloatCall(I, ISD::FLDEXP))
9580 return;
9581 break;
9582 case LibFunc_memcmp:
9583 if (visitMemCmpBCmpCall(I))
9584 return;
9585 break;
9586 case LibFunc_mempcpy:
9587 if (visitMemPCpyCall(I))
9588 return;
9589 break;
9590 case LibFunc_memchr:
9591 if (visitMemChrCall(I))
9592 return;
9593 break;
9594 case LibFunc_strcpy:
9595 if (visitStrCpyCall(I, false))
9596 return;
9597 break;
9598 case LibFunc_stpcpy:
9599 if (visitStrCpyCall(I, true))
9600 return;
9601 break;
9602 case LibFunc_strcmp:
9603 if (visitStrCmpCall(I))
9604 return;
9605 break;
9606 case LibFunc_strlen:
9607 if (visitStrLenCall(I))
9608 return;
9609 break;
9610 case LibFunc_strnlen:
9611 if (visitStrNLenCall(I))
9612 return;
9613 break;
9614 }
9615 }
9616 }
9617
9618 if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
9619 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
9620 return;
9621 }
9622
9623 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9624 // have to do anything here to lower funclet bundles.
9625 // CFGuardTarget bundles are lowered in LowerCallTo.
9627 I, "calls",
9632
9633 SDValue Callee = getValue(I.getCalledOperand());
9634
9635 if (I.hasDeoptState())
9636 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
9637 else
9638 // Check if we can potentially perform a tail call. More detailed checking
9639 // is be done within LowerCallTo, after more information about the call is
9640 // known.
9641 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
9642}
9643
9645 const CallBase &CB, const BasicBlock *EHPadBB) {
9646 auto PAB = CB.getOperandBundle("ptrauth");
9647 const Value *CalleeV = CB.getCalledOperand();
9648
9649 // Gather the call ptrauth data from the operand bundle:
9650 // [ i32 <key>, i64 <discriminator> ]
9651 const auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
9652 const Value *Discriminator = PAB->Inputs[1];
9653
9654 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
9655 assert(Discriminator->getType()->isIntegerTy(64) &&
9656 "Invalid ptrauth discriminator");
9657
9658 // Look through ptrauth constants to find the raw callee.
9659 // Do a direct unauthenticated call if we found it and everything matches.
9660 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CalleeV))
9661 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
9662 DAG.getDataLayout()))
9663 return LowerCallTo(CB, getValue(CalleeCPA->getPointer()), CB.isTailCall(),
9664 CB.isMustTailCall(), EHPadBB);
9665
9666 // Functions should never be ptrauth-called directly.
9667 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
9668
9669 // Otherwise, do an authenticated indirect call.
9670 TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
9671 getValue(Discriminator)};
9672
9673 LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
9674 EHPadBB, &PAI);
9675}
9676
9677namespace {
9678
9679/// AsmOperandInfo - This contains information for each constraint that we are
9680/// lowering.
9681class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9682public:
9683 /// CallOperand - If this is the result output operand or a clobber
9684 /// this is null, otherwise it is the incoming operand to the CallInst.
9685 /// This gets modified as the asm is processed.
9686 SDValue CallOperand;
9687
9688 /// AssignedRegs - If this is a register or register class operand, this
9689 /// contains the set of register corresponding to the operand.
9690 RegsForValue AssignedRegs;
9691
9692 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9693 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9694 }
9695
9696 /// Whether or not this operand accesses memory
9697 bool hasMemory(const TargetLowering &TLI) const {
9698 // Indirect operand accesses access memory.
9699 if (isIndirect)
9700 return true;
9701
9702 for (const auto &Code : Codes)
9704 return true;
9705
9706 return false;
9707 }
9708};
9709
9710
9711} // end anonymous namespace
9712
9713/// Make sure that the output operand \p OpInfo and its corresponding input
9714/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9715/// out).
9716static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9717 SDISelAsmOperandInfo &MatchingOpInfo,
9718 SelectionDAG &DAG) {
9719 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9720 return;
9721
9723 const auto &TLI = DAG.getTargetLoweringInfo();
9724
9725 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9726 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
9727 OpInfo.ConstraintVT);
9728 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9729 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
9730 MatchingOpInfo.ConstraintVT);
9731 const bool OutOpIsIntOrFP =
9732 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
9733 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
9734 MatchingOpInfo.ConstraintVT.isFloatingPoint();
9735 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
9736 // FIXME: error out in a more elegant fashion
9737 report_fatal_error("Unsupported asm: input constraint"
9738 " with a matching output constraint of"
9739 " incompatible type!");
9740 }
9741 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9742}
9743
9744/// Get a direct memory input to behave well as an indirect operand.
9745/// This may introduce stores, hence the need for a \p Chain.
9746/// \return The (possibly updated) chain.
9747static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9748 SDISelAsmOperandInfo &OpInfo,
9749 SelectionDAG &DAG) {
9750 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9751
9752 // If we don't have an indirect input, put it in the constpool if we can,
9753 // otherwise spill it to a stack slot.
9754 // TODO: This isn't quite right. We need to handle these according to
9755 // the addressing mode that the constraint wants. Also, this may take
9756 // an additional register for the computation and we don't want that
9757 // either.
9758
9759 // If the operand is a float, integer, or vector constant, spill to a
9760 // constant pool entry to get its address.
9761 const Value *OpVal = OpInfo.CallOperandVal;
9762 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
9764 OpInfo.CallOperand = DAG.getConstantPool(
9765 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
9766 return Chain;
9767 }
9768
9769 // Otherwise, create a stack slot and emit a store to it before the asm.
9770 Type *Ty = OpVal->getType();
9771 auto &DL = DAG.getDataLayout();
9772 TypeSize TySize = DL.getTypeAllocSize(Ty);
9775 int StackID = 0;
9776 if (TySize.isScalable())
9777 StackID = TFI->getStackIDForScalableVectors();
9778 int SSFI = MF.getFrameInfo().CreateStackObject(TySize.getKnownMinValue(),
9779 DL.getPrefTypeAlign(Ty), false,
9780 nullptr, StackID);
9781 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
9782 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
9784 TLI.getMemValueType(DL, Ty));
9785 OpInfo.CallOperand = StackSlot;
9786
9787 return Chain;
9788}
9789
9790/// GetRegistersForValue - Assign registers (virtual or physical) for the
9791/// specified operand. We prefer to assign virtual registers, to allow the
9792/// register allocator to handle the assignment process. However, if the asm
9793/// uses features that we can't model on machineinstrs, we have SDISel do the
9794/// allocation. This produces generally horrible, but correct, code.
9795///
9796/// OpInfo describes the operand
9797/// RefOpInfo describes the matching operand if any, the operand otherwise
9798static std::optional<unsigned>
9800 SDISelAsmOperandInfo &OpInfo,
9801 SDISelAsmOperandInfo &RefOpInfo) {
9802 LLVMContext &Context = *DAG.getContext();
9803 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9804
9808
9809 // No work to do for memory/address operands.
9810 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9811 OpInfo.ConstraintType == TargetLowering::C_Address)
9812 return std::nullopt;
9813
9814 // If this is a constraint for a single physreg, or a constraint for a
9815 // register class, find it.
9816 unsigned AssignedReg;
9817 const TargetRegisterClass *RC;
9818 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
9819 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
9820 // RC is unset only on failure. Return immediately.
9821 if (!RC)
9822 return std::nullopt;
9823
9824 // Get the actual register value type. This is important, because the user
9825 // may have asked for (e.g.) the AX register in i32 type. We need to
9826 // remember that AX is actually i16 to get the right extension.
9827 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
9828
9829 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9830 // If this is an FP operand in an integer register (or visa versa), or more
9831 // generally if the operand value disagrees with the register class we plan
9832 // to stick it in, fix the operand type.
9833 //
9834 // If this is an input value, the bitcast to the new type is done now.
9835 // Bitcast for output value is done at the end of visitInlineAsm().
9836 if ((OpInfo.Type == InlineAsm::isOutput ||
9837 OpInfo.Type == InlineAsm::isInput) &&
9838 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
9839 // Try to convert to the first EVT that the reg class contains. If the
9840 // types are identical size, use a bitcast to convert (e.g. two differing
9841 // vector types). Note: output bitcast is done at the end of
9842 // visitInlineAsm().
9843 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9844 // Exclude indirect inputs while they are unsupported because the code
9845 // to perform the load is missing and thus OpInfo.CallOperand still
9846 // refers to the input address rather than the pointed-to value.
9847 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
9848 OpInfo.CallOperand =
9849 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
9850 OpInfo.ConstraintVT = RegVT;
9851 // If the operand is an FP value and we want it in integer registers,
9852 // use the corresponding integer type. This turns an f64 value into
9853 // i64, which can be passed with two i32 values on a 32-bit machine.
9854 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9855 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
9856 if (OpInfo.Type == InlineAsm::isInput)
9857 OpInfo.CallOperand =
9858 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
9859 OpInfo.ConstraintVT = VT;
9860 }
9861 }
9862 }
9863
9864 // No need to allocate a matching input constraint since the constraint it's
9865 // matching to has already been allocated.
9866 if (OpInfo.isMatchingInputConstraint())
9867 return std::nullopt;
9868
9869 EVT ValueVT = OpInfo.ConstraintVT;
9870 if (OpInfo.ConstraintVT == MVT::Other)
9871 ValueVT = RegVT;
9872
9873 // Initialize NumRegs.
9874 unsigned NumRegs = 1;
9875 if (OpInfo.ConstraintVT != MVT::Other)
9876 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
9877
9878 // If this is a constraint for a specific physical register, like {r17},
9879 // assign it now.
9880
9881 // If this associated to a specific register, initialize iterator to correct
9882 // place. If virtual, make sure we have enough registers
9883
9884 // Initialize iterator if necessary
9887
9888 // Do not check for single registers.
9889 if (AssignedReg) {
9890 I = std::find(I, RC->end(), AssignedReg);
9891 if (I == RC->end()) {
9892 // RC does not contain the selected register, which indicates a
9893 // mismatch between the register and the required type/bitwidth.
9894 return {AssignedReg};
9895 }
9896 }
9897
9898 for (; NumRegs; --NumRegs, ++I) {
9899 assert(I != RC->end() && "Ran out of registers to allocate!");
9900 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
9901 Regs.push_back(R);
9902 }
9903
9904 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
9905 return std::nullopt;
9906}
9907
9908static unsigned
9910 const std::vector<SDValue> &AsmNodeOperands) {
9911 // Scan until we find the definition we already emitted of this operand.
9912 unsigned CurOp = InlineAsm::Op_FirstOperand;
9913 for (; OperandNo; --OperandNo) {
9914 // Advance to the next operand.
9915 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9916 const InlineAsm::Flag F(OpFlag);
9917 assert(
9918 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
9919 "Skipped past definitions?");
9920 CurOp += F.getNumOperandRegisters() + 1;
9921 }
9922 return CurOp;
9923}
9924
9925namespace {
9926
9927class ExtraFlags {
9928 unsigned Flags = 0;
9929
9930public:
9931 explicit ExtraFlags(const CallBase &Call) {
9932 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9933 if (IA->hasSideEffects())
9935 if (IA->isAlignStack())
9937 if (Call.isConvergent())
9939 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
9940 }
9941
9942 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
9943 // Ideally, we would only check against memory constraints. However, the
9944 // meaning of an Other constraint can be target-specific and we can't easily
9945 // reason about it. Therefore, be conservative and set MayLoad/MayStore
9946 // for Other constraints as well.
9949 if (OpInfo.Type == InlineAsm::isInput)
9951 else if (OpInfo.Type == InlineAsm::isOutput)
9953 else if (OpInfo.Type == InlineAsm::isClobber)
9955 }
9956 }
9957
9958 unsigned get() const { return Flags; }
9959};
9960
9961} // end anonymous namespace
9962
9963static bool isFunction(SDValue Op) {
9964 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
9965 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
9966 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
9967
9968 // In normal "call dllimport func" instruction (non-inlineasm) it force
9969 // indirect access by specifing call opcode. And usually specially print
9970 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
9971 // not do in this way now. (In fact, this is similar with "Data Access"
9972 // action). So here we ignore dllimport function.
9973 if (Fn && !Fn->hasDLLImportStorageClass())
9974 return true;
9975 }
9976 }
9977 return false;
9978}
9979
9980/// visitInlineAsm - Handle a call to an InlineAsm object.
9981void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
9982 const BasicBlock *EHPadBB) {
9983 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9984
9985 /// ConstraintOperands - Information about all of the constraints.
9986 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
9987
9988 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9990 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);
9991
9992 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
9993 // AsmDialect, MayLoad, MayStore).
9994 bool HasSideEffect = IA->hasSideEffects();
9995 ExtraFlags ExtraInfo(Call);
9996
9997 for (auto &T : TargetConstraints) {
9998 ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
9999 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
10000
10001 if (OpInfo.CallOperandVal)
10002 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
10003
10004 if (!HasSideEffect)
10005 HasSideEffect = OpInfo.hasMemory(TLI);
10006
10007 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
10008 // FIXME: Could we compute this on OpInfo rather than T?
10009
10010 // Compute the constraint code and ConstraintType to use.
10012
10013 if (T.ConstraintType == TargetLowering::C_Immediate &&
10014 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
10015 // We've delayed emitting a diagnostic like the "n" constraint because
10016 // inlining could cause an integer showing up.
10017 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
10018 "' expects an integer constant "
10019 "expression");
10020
10021 ExtraInfo.update(T);
10022 }
10023
10024 // We won't need to flush pending loads if this asm doesn't touch
10025 // memory and is nonvolatile.
10026 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
10027
10028 bool EmitEHLabels = isa<InvokeInst>(Call);
10029 if (EmitEHLabels) {
10030 assert(EHPadBB && "InvokeInst must have an EHPadBB");
10031 }
10032 bool IsCallBr = isa<CallBrInst>(Call);
10033
10034 if (IsCallBr || EmitEHLabels) {
10035 // If this is a callbr or invoke we need to flush pending exports since
10036 // inlineasm_br and invoke are terminators.
10037 // We need to do this before nodes are glued to the inlineasm_br node.
10038 Chain = getControlRoot();
10039 }
10040
10041 MCSymbol *BeginLabel = nullptr;
10042 if (EmitEHLabels) {
10043 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
10044 }
10045
10046 int OpNo = -1;
10047 SmallVector<StringRef> AsmStrs;
10048 IA->collectAsmStrs(AsmStrs);
10049
10050 // Second pass over the constraints: compute which constraint option to use.
10051 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10052 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10053 OpNo++;
10054
10055 // If this is an output operand with a matching input operand, look up the
10056 // matching input. If their types mismatch, e.g. one is an integer, the
10057 // other is floating point, or their sizes are different, flag it as an
10058 // error.
10059 if (OpInfo.hasMatchingInput()) {
10060 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
10061 patchMatchingInput(OpInfo, Input, DAG);
10062 }
10063
10064 // Compute the constraint code and ConstraintType to use.
10065 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
10066
10067 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10068 OpInfo.Type == InlineAsm::isClobber) ||
10069 OpInfo.ConstraintType == TargetLowering::C_Address)
10070 continue;
10071
10072 // In Linux PIC model, there are 4 cases about value/label addressing:
10073 //
10074 // 1: Function call or Label jmp inside the module.
10075 // 2: Data access (such as global variable, static variable) inside module.
10076 // 3: Function call or Label jmp outside the module.
10077 // 4: Data access (such as global variable) outside the module.
10078 //
10079 // Due to current llvm inline asm architecture designed to not "recognize"
10080 // the asm code, there are quite troubles for us to treat mem addressing
10081 // differently for same value/adress used in different instuctions.
10082 // For example, in pic model, call a func may in plt way or direclty
10083 // pc-related, but lea/mov a function adress may use got.
10084 //
10085 // Here we try to "recognize" function call for the case 1 and case 3 in
10086 // inline asm. And try to adjust the constraint for them.
10087 //
10088 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10089 // label, so here we don't handle jmp function label now, but we need to
10090 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10091 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
10092 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10093 TM.getCodeModel() != CodeModel::Large) {
10094 OpInfo.isIndirect = false;
10095 OpInfo.ConstraintType = TargetLowering::C_Address;
10096 }
10097
10098 // If this is a memory input, and if the operand is not indirect, do what we
10099 // need to provide an address for the memory input.
10100 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10101 !OpInfo.isIndirect) {
10102 assert((OpInfo.isMultipleAlternative ||
10103 (OpInfo.Type == InlineAsm::isInput)) &&
10104 "Can only indirectify direct input operands!");
10105
10106 // Memory operands really want the address of the value.
10107 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
10108
10109 // There is no longer a Value* corresponding to this operand.
10110 OpInfo.CallOperandVal = nullptr;
10111
10112 // It is now an indirect operand.
10113 OpInfo.isIndirect = true;
10114 }
10115
10116 }
10117
10118 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10119 std::vector<SDValue> AsmNodeOperands;
10120 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
10121 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
10122 IA->getAsmString().data(), TLI.getProgramPointerTy(DAG.getDataLayout())));
10123
10124 // If we have a !srcloc metadata node associated with it, we want to attach
10125 // this to the ultimately generated inline asm machineinstr. To do this, we
10126 // pass in the third operand as this (potentially null) inline asm MDNode.
10127 const MDNode *SrcLoc = Call.getMetadata("srcloc");
10128 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
10129
10130 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10131 // bits as operand 3.
10132 AsmNodeOperands.push_back(DAG.getTargetConstant(
10133 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
10134
10135 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
10136 // this, assign virtual and physical registers for inputs and otput.
10137 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10138 // Assign Registers.
10139 SDISelAsmOperandInfo &RefOpInfo =
10140 OpInfo.isMatchingInputConstraint()
10141 ? ConstraintOperands[OpInfo.getMatchedOperand()]
10142 : OpInfo;
10143 const auto RegError =
10144 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
10145 if (RegError) {
10146 const MachineFunction &MF = DAG.getMachineFunction();
10147 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10148 const char *RegName = TRI.getName(*RegError);
10149 emitInlineAsmError(Call, "register '" + Twine(RegName) +
10150 "' allocated for constraint '" +
10151 Twine(OpInfo.ConstraintCode) +
10152 "' does not match required type");
10153 return;
10154 }
10155
10156 auto DetectWriteToReservedRegister = [&]() {
10157 const MachineFunction &MF = DAG.getMachineFunction();
10158 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10159 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10160 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
10161 const char *RegName = TRI.getName(Reg);
10162 emitInlineAsmError(Call, "write to reserved register '" +
10163 Twine(RegName) + "'");
10164 return true;
10165 }
10166 }
10167 return false;
10168 };
10169 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10170 (OpInfo.Type == InlineAsm::isInput &&
10171 !OpInfo.isMatchingInputConstraint())) &&
10172 "Only address as input operand is allowed.");
10173
10174 switch (OpInfo.Type) {
10176 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10177 const InlineAsm::ConstraintCode ConstraintID =
10178 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10180 "Failed to convert memory constraint code to constraint id.");
10181
10182 // Add information to the INLINEASM node to know about this output.
10183 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
10184 OpFlags.setMemConstraint(ConstraintID);
10185 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
10186 MVT::i32));
10187 AsmNodeOperands.push_back(OpInfo.CallOperand);
10188 } else {
10189 // Otherwise, this outputs to a register (directly for C_Register /
10190 // C_RegisterClass, and a target-defined fashion for
10191 // C_Immediate/C_Other). Find a register that we can use.
10192 if (OpInfo.AssignedRegs.Regs.empty()) {
10193 emitInlineAsmError(
10194 Call, "couldn't allocate output register for constraint '" +
10195 Twine(OpInfo.ConstraintCode) + "'");
10196 return;
10197 }
10198
10199 if (DetectWriteToReservedRegister())
10200 return;
10201
10202 // Add information to the INLINEASM node to know that this register is
10203 // set.
10204 OpInfo.AssignedRegs.AddInlineAsmOperands(
10205 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10207 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
10208 }
10209 break;
10210
10211 case InlineAsm::isInput:
10212 case InlineAsm::isLabel: {
10213 SDValue InOperandVal = OpInfo.CallOperand;
10214
10215 if (OpInfo.isMatchingInputConstraint()) {
10216 // If this is required to match an output register we have already set,
10217 // just use its register.
10218 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
10219 AsmNodeOperands);
10220 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
10221 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10222 if (OpInfo.isIndirect) {
10223 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10224 emitInlineAsmError(Call, "inline asm not supported yet: "
10225 "don't know how to handle tied "
10226 "indirect register inputs");
10227 return;
10228 }
10229
10231 MachineFunction &MF = DAG.getMachineFunction();
10232 MachineRegisterInfo &MRI = MF.getRegInfo();
10233 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
10234 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
10235 Register TiedReg = R->getReg();
10236 MVT RegVT = R->getSimpleValueType(0);
10237 const TargetRegisterClass *RC =
10238 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)
10239 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
10240 : TRI.getMinimalPhysRegClass(TiedReg);
10241 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
10242 Regs.push_back(MRI.createVirtualRegister(RC));
10243
10244 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10245
10246 SDLoc dl = getCurSDLoc();
10247 // Use the produced MatchedRegs object to
10248 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);
10249 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, true,
10250 OpInfo.getMatchedOperand(), dl, DAG,
10251 AsmNodeOperands);
10252 break;
10253 }
10254
10255 assert(Flag.isMemKind() && "Unknown matching constraint!");
10256 assert(Flag.getNumOperandRegisters() == 1 &&
10257 "Unexpected number of operands");
10258 // Add information to the INLINEASM node to know about this input.
10259 // See InlineAsm.h isUseOperandTiedToDef.
10260 Flag.clearMemConstraint();
10261 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10262 AsmNodeOperands.push_back(DAG.getTargetConstant(
10263 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
10264 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
10265 break;
10266 }
10267
10268 // Treat indirect 'X' constraint as memory.
10269 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10270 OpInfo.isIndirect)
10271 OpInfo.ConstraintType = TargetLowering::C_Memory;
10272
10273 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10274 OpInfo.ConstraintType == TargetLowering::C_Other) {
10275 std::vector<SDValue> Ops;
10276 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
10277 Ops, DAG);
10278 if (Ops.empty()) {
10279 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10280 if (isa<ConstantSDNode>(InOperandVal)) {
10281 emitInlineAsmError(Call, "value out of range for constraint '" +
10282 Twine(OpInfo.ConstraintCode) + "'");
10283 return;
10284 }
10285
10286 emitInlineAsmError(Call,
10287 "invalid operand for inline asm constraint '" +
10288 Twine(OpInfo.ConstraintCode) + "'");
10289 return;
10290 }
10291
10292 // Add information to the INLINEASM node to know about this input.
10293 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10294 AsmNodeOperands.push_back(DAG.getTargetConstant(
10295 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
10296 llvm::append_range(AsmNodeOperands, Ops);
10297 break;
10298 }
10299
10300 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10301 assert((OpInfo.isIndirect ||
10302 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10303 "Operand must be indirect to be a mem!");
10304 assert(InOperandVal.getValueType() ==
10305 TLI.getPointerTy(DAG.getDataLayout()) &&
10306 "Memory operands expect pointer values");
10307
10308 const InlineAsm::ConstraintCode ConstraintID =
10309 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10311 "Failed to convert memory constraint code to constraint id.");
10312
10313 // Add information to the INLINEASM node to know about this input.
10314 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10315 ResOpType.setMemConstraint(ConstraintID);
10316 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
10317 getCurSDLoc(),
10318 MVT::i32));
10319 AsmNodeOperands.push_back(InOperandVal);
10320 break;
10321 }
10322
10323 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10324 const InlineAsm::ConstraintCode ConstraintID =
10325 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10327 "Failed to convert memory constraint code to constraint id.");
10328
10329 InlineAsm::Flag ResOpType(InlineAsm::Kind::Mem, 1);
10330
10331 SDValue AsmOp = InOperandVal;
10332 if (isFunction(InOperandVal)) {
10333 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
10334 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10335 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
10336 InOperandVal.getValueType(),
10337 GA->getOffset());
10338 }
10339
10340 // Add information to the INLINEASM node to know about this input.
10341 ResOpType.setMemConstraint(ConstraintID);
10342
10343 AsmNodeOperands.push_back(
10344 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
10345
10346 AsmNodeOperands.push_back(AsmOp);
10347 break;
10348 }
10349
10350 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10351 OpInfo.ConstraintType != TargetLowering::C_Register) {
10352 emitInlineAsmError(Call, "unknown asm constraint '" +
10353 Twine(OpInfo.ConstraintCode) + "'");
10354 return;
10355 }
10356
10357 // TODO: Support this.
10358 if (OpInfo.isIndirect) {
10359 emitInlineAsmError(
10360 Call, "Don't know how to handle indirect register inputs yet "
10361 "for constraint '" +
10362 Twine(OpInfo.ConstraintCode) + "'");
10363 return;
10364 }
10365
10366 // Copy the input into the appropriate registers.
10367 if (OpInfo.AssignedRegs.Regs.empty()) {
10368 emitInlineAsmError(Call,
10369 "couldn't allocate input reg for constraint '" +
10370 Twine(OpInfo.ConstraintCode) + "'");
10371 return;
10372 }
10373
10374 if (DetectWriteToReservedRegister())
10375 return;
10376
10377 SDLoc dl = getCurSDLoc();
10378
10379 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,
10380 &Call);
10381
10382 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, false,
10383 0, dl, DAG, AsmNodeOperands);
10384 break;
10385 }
10387 // Add the clobbered value to the operand list, so that the register
10388 // allocator is aware that the physreg got clobbered.
10389 if (!OpInfo.AssignedRegs.Regs.empty())
10391 false, 0, getCurSDLoc(), DAG,
10392 AsmNodeOperands);
10393 break;
10394 }
10395 }
10396
10397 // Finish up input operands. Set the input chain and add the flag last.
10398 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10399 if (Glue.getNode()) AsmNodeOperands.push_back(Glue);
10400
10401 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10402 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
10403 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
10404 Glue = Chain.getValue(1);
10405
10406 // Do additional work to generate outputs.
10407
10408 SmallVector<EVT, 1> ResultVTs;
10409 SmallVector<SDValue, 1> ResultValues;
10410 SmallVector<SDValue, 8> OutChains;
10411
10412 llvm::Type *CallResultType = Call.getType();
10413 ArrayRef<Type *> ResultTypes;
10414 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
10415 ResultTypes = StructResult->elements();
10416 else if (!CallResultType->isVoidTy())
10417 ResultTypes = ArrayRef(CallResultType);
10418
10419 auto CurResultType = ResultTypes.begin();
10420 auto handleRegAssign = [&](SDValue V) {
10421 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10422 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10423 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
10424 ++CurResultType;
10425 // If the type of the inline asm call site return value is different but has
10426 // same size as the type of the asm output bitcast it. One example of this
10427 // is for vectors with different width / number of elements. This can
10428 // happen for register classes that can contain multiple different value
10429 // types. The preg or vreg allocated may not have the same VT as was
10430 // expected.
10431 //
10432 // This can also happen for a return value that disagrees with the register
10433 // class it is put in, eg. a double in a general-purpose register on a
10434 // 32-bit machine.
10435 if (ResultVT != V.getValueType() &&
10436 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10437 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
10438 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10439 V.getValueType().isInteger()) {
10440 // If a result value was tied to an input value, the computed result
10441 // may have a wider width than the expected result. Extract the
10442 // relevant portion.
10443 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
10444 }
10445 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10446 ResultVTs.push_back(ResultVT);
10447 ResultValues.push_back(V);
10448 };
10449
10450 // Deal with output operands.
10451 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10452 if (OpInfo.Type == InlineAsm::isOutput) {
10453 SDValue Val;
10454 // Skip trivial output operands.
10455 if (OpInfo.AssignedRegs.Regs.empty())
10456 continue;
10457
10458 switch (OpInfo.ConstraintType) {
10461 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
10462 Chain, &Glue, &Call);
10463 break;
10466 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
10467 OpInfo, DAG);
10468 break;
10470 break; // Already handled.
10472 break; // Silence warning.
10474 assert(false && "Unexpected unknown constraint");
10475 }
10476
10477 // Indirect output manifest as stores. Record output chains.
10478 if (OpInfo.isIndirect) {
10479 const Value *Ptr = OpInfo.CallOperandVal;
10480 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10481 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
10482 MachinePointerInfo(Ptr));
10483 OutChains.push_back(Store);
10484 } else {
10485 // generate CopyFromRegs to associated registers.
10486 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10487 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10488 for (const SDValue &V : Val->op_values())
10489 handleRegAssign(V);
10490 } else
10491 handleRegAssign(Val);
10492 }
10493 }
10494 }
10495
10496 // Set results.
10497 if (!ResultValues.empty()) {
10498 assert(CurResultType == ResultTypes.end() &&
10499 "Mismatch in number of ResultTypes");
10500 assert(ResultValues.size() == ResultTypes.size() &&
10501 "Mismatch in number of output operands in asm result");
10502
10504 DAG.getVTList(ResultVTs), ResultValues);
10505 setValue(&Call, V);
10506 }
10507
10508 // Collect store chains.
10509 if (!OutChains.empty())
10510 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
10511
10512 if (EmitEHLabels) {
10513 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
10514 }
10515
10516 // Only Update Root if inline assembly has a memory effect.
10517 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10518 EmitEHLabels)
10519 DAG.setRoot(Chain);
10520}
10521
10522void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10523 const Twine &Message) {
10524 LLVMContext &Ctx = *DAG.getContext();
10525 Ctx.diagnose(DiagnosticInfoInlineAsm(Call, Message));
10526
10527 // Make sure we leave the DAG in a valid state
10528 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10529 SmallVector<EVT, 1> ValueVTs;
10530 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
10531
10532 if (ValueVTs.empty())
10533 return;
10534
10536 for (const EVT &VT : ValueVTs)
10537 Ops.push_back(DAG.getUNDEF(VT));
10538
10539 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
10540}
10541
10542void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10543 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
10544 MVT::Other, getRoot(),
10545 getValue(I.getArgOperand(0)),
10546 DAG.getSrcValue(I.getArgOperand(0))));
10547}
10548
10549void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10550 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10551 const DataLayout &DL = DAG.getDataLayout();
10552 SDValue V = DAG.getVAArg(
10553 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
10554 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
10555 DL.getABITypeAlign(I.getType()).value());
10556 DAG.setRoot(V.getValue(1));
10557
10558 if (I.getType()->isPointerTy())
10559 V = DAG.getPtrExtOrTrunc(
10560 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
10561 setValue(&I, V);
10562}
10563
10564void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10565 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
10566 MVT::Other, getRoot(),
10567 getValue(I.getArgOperand(0)),
10568 DAG.getSrcValue(I.getArgOperand(0))));
10569}
10570
10571void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10572 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
10573 MVT::Other, getRoot(),
10574 getValue(I.getArgOperand(0)),
10575 getValue(I.getArgOperand(1)),
10576 DAG.getSrcValue(I.getArgOperand(0)),
10577 DAG.getSrcValue(I.getArgOperand(1))));
10578}
10579
10581 const Instruction &I,
10582 SDValue Op) {
10583 std::optional<ConstantRange> CR = getRange(I);
10584
10585 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
10586 return Op;
10587
10588 APInt Lo = CR->getUnsignedMin();
10589 if (!Lo.isMinValue())
10590 return Op;
10591
10592 APInt Hi = CR->getUnsignedMax();
10593 unsigned Bits = std::max(Hi.getActiveBits(),
10594 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10595
10596 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
10597
10598 SDLoc SL = getCurSDLoc();
10599
10600 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
10601 DAG.getValueType(SmallVT));
10602 unsigned NumVals = Op.getNode()->getNumValues();
10603 if (NumVals == 1)
10604 return ZExt;
10605
10607
10608 Ops.push_back(ZExt);
10609 for (unsigned I = 1; I != NumVals; ++I)
10610 Ops.push_back(Op.getValue(I));
10611
10612 return DAG.getMergeValues(Ops, SL);
10613}
10614
10615/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10616/// the call being lowered.
10617///
10618/// This is a helper for lowering intrinsics that follow a target calling
10619/// convention or require stack pointer adjustment. Only a subset of the
10620/// intrinsic's operands need to participate in the calling convention.
10623 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10624 AttributeSet RetAttrs, bool IsPatchPoint) {
10626 Args.reserve(NumArgs);
10627
10628 // Populate the argument list.
10629 // Attributes for args start at offset 1, after the return attribute.
10630 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10631 ArgI != ArgE; ++ArgI) {
10632 const Value *V = Call->getOperand(ArgI);
10633
10634 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10635
10636 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
10637 Entry.setAttributes(Call, ArgI);
10638 Args.push_back(Entry);
10639 }
10640
10642 .setChain(getRoot())
10643 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
10644 RetAttrs)
10645 .setDiscardResult(Call->use_empty())
10646 .setIsPatchPoint(IsPatchPoint)
10648 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
10649}
10650
10651/// Add a stack map intrinsic call's live variable operands to a stackmap
10652/// or patchpoint target node's operand list.
10653///
10654/// Constants are converted to TargetConstants purely as an optimization to
10655/// avoid constant materialization and register allocation.
10656///
10657/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10658/// generate addess computation nodes, and so FinalizeISel can convert the
10659/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10660/// address materialization and register allocation, but may also be required
10661/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10662/// alloca in the entry block, then the runtime may assume that the alloca's
10663/// StackMap location can be read immediately after compilation and that the
10664/// location is valid at any point during execution (this is similar to the
10665/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10666/// only available in a register, then the runtime would need to trap when
10667/// execution reaches the StackMap in order to read the alloca's location.
10668static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10670 SelectionDAGBuilder &Builder) {
10671 SelectionDAG &DAG = Builder.DAG;
10672 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10673 SDValue Op = Builder.getValue(Call.getArgOperand(I));
10674
10675 // Things on the stack are pointer-typed, meaning that they are already
10676 // legal and can be emitted directly to target nodes.
10678 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
10679 } else {
10680 // Otherwise emit a target independent node to be legalised.
10681 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
10682 }
10683 }
10684}
10685
10686/// Lower llvm.experimental.stackmap.
10687void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10688 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10689 // [live variables...])
10690
10691 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10692
10693 SDValue Chain, InGlue, Callee;
10695
10696 SDLoc DL = getCurSDLoc();
10698
10699 // The stackmap intrinsic only records the live variables (the arguments
10700 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10701 // intrinsic, this won't be lowered to a function call. This means we don't
10702 // have to worry about calling conventions and target specific lowering code.
10703 // Instead we perform the call lowering right here.
10704 //
10705 // chain, flag = CALLSEQ_START(chain, 0, 0)
10706 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10707 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10708 //
10709 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
10710 InGlue = Chain.getValue(1);
10711
10712 // Add the STACKMAP operands, starting with DAG house-keeping.
10713 Ops.push_back(Chain);
10714 Ops.push_back(InGlue);
10715
10716 // Add the <id>, <numShadowBytes> operands.
10717 //
10718 // These do not require legalisation, and can be emitted directly to target
10719 // constant nodes.
10721 assert(ID.getValueType() == MVT::i64);
10722 SDValue IDConst =
10723 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
10724 Ops.push_back(IDConst);
10725
10726 SDValue Shad = getValue(CI.getArgOperand(1));
10727 assert(Shad.getValueType() == MVT::i32);
10728 SDValue ShadConst =
10729 DAG.getTargetConstant(Shad->getAsZExtVal(), DL, Shad.getValueType());
10730 Ops.push_back(ShadConst);
10731
10732 // Add the live variables.
10733 addStackMapLiveVars(CI, 2, DL, Ops, *this);
10734
10735 // Create the STACKMAP node.
10736 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10737 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
10738 InGlue = Chain.getValue(1);
10739
10740 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
10741
10742 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10743
10744 // Set the root to the target-lowered call chain.
10745 DAG.setRoot(Chain);
10746
10747 // Inform the Frame Information that we have a stackmap in this function.
10748 FuncInfo.MF->getFrameInfo().setHasStackMap();
10749}
10750
10751/// Lower llvm.experimental.patchpoint directly to its target opcode.
10752void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10753 const BasicBlock *EHPadBB) {
10754 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
10755 // i32 <numBytes>,
10756 // i8* <target>,
10757 // i32 <numArgs>,
10758 // [Args...],
10759 // [live variables...])
10760
10762 bool IsAnyRegCC = CC == CallingConv::AnyReg;
10763 bool HasDef = !CB.getType()->isVoidTy();
10764 SDLoc dl = getCurSDLoc();
10766
10767 // Handle immediate and symbolic callees.
10768 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
10769 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
10770 /*isTarget=*/true);
10771 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
10772 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
10773 SDLoc(SymbolicCallee),
10774 SymbolicCallee->getValueType(0));
10775
10776 // Get the real number of arguments participating in the call <numArgs>
10778 unsigned NumArgs = NArgVal->getAsZExtVal();
10779
10780 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
10781 // Intrinsics include all meta-operands up to but not including CC.
10782 unsigned NumMetaOpers = PatchPointOpers::CCPos;
10783 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
10784 "Not enough arguments provided to the patchpoint intrinsic");
10785
10786 // For AnyRegCC the arguments are lowered later on manually.
10787 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10788 Type *ReturnTy =
10789 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
10790
10791 TargetLowering::CallLoweringInfo CLI(DAG);
10792 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
10793 ReturnTy, CB.getAttributes().getRetAttrs(), true);
10794 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
10795
10796 SDNode *CallEnd = Result.second.getNode();
10797 if (CallEnd->getOpcode() == ISD::EH_LABEL)
10798 CallEnd = CallEnd->getOperand(0).getNode();
10799 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
10800 CallEnd = CallEnd->getOperand(0).getNode();
10801
10802 /// Get a call instruction from the call sequence chain.
10803 /// Tail calls are not allowed.
10804 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
10805 "Expected a callseq node.");
10806 SDNode *Call = CallEnd->getOperand(0).getNode();
10807 bool HasGlue = Call->getGluedNode();
10808
10809 // Replace the target specific call node with the patchable intrinsic.
10811
10812 // Push the chain.
10813 Ops.push_back(*(Call->op_begin()));
10814
10815 // Optionally, push the glue (if any).
10816 if (HasGlue)
10817 Ops.push_back(*(Call->op_end() - 1));
10818
10819 // Push the register mask info.
10820 if (HasGlue)
10821 Ops.push_back(*(Call->op_end() - 2));
10822 else
10823 Ops.push_back(*(Call->op_end() - 1));
10824
10825 // Add the <id> and <numBytes> constants.
10827 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
10829 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
10830
10831 // Add the callee.
10832 Ops.push_back(Callee);
10833
10834 // Adjust <numArgs> to account for any arguments that have been passed on the
10835 // stack instead.
10836 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
10837 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
10838 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10839 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
10840
10841 // Add the calling convention
10842 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
10843
10844 // Add the arguments we omitted previously. The register allocator should
10845 // place these in any free register.
10846 if (IsAnyRegCC)
10847 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
10848 Ops.push_back(getValue(CB.getArgOperand(i)));
10849
10850 // Push the arguments from the call instruction.
10851 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
10852 Ops.append(Call->op_begin() + 2, e);
10853
10854 // Push live variables for the stack map.
10855 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
10856
10857 SDVTList NodeTys;
10858 if (IsAnyRegCC && HasDef) {
10859 // Create the return types based on the intrinsic definition
10860 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10861 SmallVector<EVT, 3> ValueVTs;
10862 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
10863 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
10864
10865 // There is always a chain and a glue type at the end
10866 ValueVTs.push_back(MVT::Other);
10867 ValueVTs.push_back(MVT::Glue);
10868 NodeTys = DAG.getVTList(ValueVTs);
10869 } else
10870 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10871
10872 // Replace the target specific call node with a PATCHPOINT node.
10873 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
10874
10875 // Update the NodeMap.
10876 if (HasDef) {
10877 if (IsAnyRegCC)
10878 setValue(&CB, SDValue(PPV.getNode(), 0));
10879 else
10880 setValue(&CB, Result.first);
10881 }
10882
10883 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
10884 // call sequence. Furthermore the location of the chain and glue can change
10885 // when the AnyReg calling convention is used and the intrinsic returns a
10886 // value.
10887 if (IsAnyRegCC && HasDef) {
10888 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
10889 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
10890 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
10891 } else
10892 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
10893 DAG.DeleteNode(Call);
10894
10895 // Inform the Frame Information that we have a patchpoint in this function.
10896 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
10897}
10898
10899void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
10900 unsigned Intrinsic) {
10901 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10902 SDValue Op1 = getValue(I.getArgOperand(0));
10903 SDValue Op2;
10904 if (I.arg_size() > 1)
10905 Op2 = getValue(I.getArgOperand(1));
10906 SDLoc dl = getCurSDLoc();
10907 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
10908 SDValue Res;
10909 SDNodeFlags SDFlags;
10910 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
10911 SDFlags.copyFMF(*FPMO);
10912
10913 switch (Intrinsic) {
10914 case Intrinsic::vector_reduce_fadd:
10915 if (SDFlags.hasAllowReassociation())
10916 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
10917 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
10918 SDFlags);
10919 else
10920 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
10921 break;
10922 case Intrinsic::vector_reduce_fmul:
10923 if (SDFlags.hasAllowReassociation())
10924 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
10925 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
10926 SDFlags);
10927 else
10928 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
10929 break;
10930 case Intrinsic::vector_reduce_add:
10931 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
10932 break;
10933 case Intrinsic::vector_reduce_mul:
10934 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
10935 break;
10936 case Intrinsic::vector_reduce_and:
10937 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
10938 break;
10939 case Intrinsic::vector_reduce_or:
10940 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
10941 break;
10942 case Intrinsic::vector_reduce_xor:
10943 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
10944 break;
10945 case Intrinsic::vector_reduce_smax:
10946 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
10947 break;
10948 case Intrinsic::vector_reduce_smin:
10949 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
10950 break;
10951 case Intrinsic::vector_reduce_umax:
10952 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
10953 break;
10954 case Intrinsic::vector_reduce_umin:
10955 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
10956 break;
10957 case Intrinsic::vector_reduce_fmax:
10958 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
10959 break;
10960 case Intrinsic::vector_reduce_fmin:
10961 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
10962 break;
10963 case Intrinsic::vector_reduce_fmaximum:
10964 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
10965 break;
10966 case Intrinsic::vector_reduce_fminimum:
10967 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
10968 break;
10969 default:
10970 llvm_unreachable("Unhandled vector reduce intrinsic");
10971 }
10972 setValue(&I, Res);
10973}
10974
10975/// Returns an AttributeList representing the attributes applied to the return
10976/// value of the given call.
10979 if (CLI.RetSExt)
10980 Attrs.push_back(Attribute::SExt);
10981 if (CLI.RetZExt)
10982 Attrs.push_back(Attribute::ZExt);
10983 if (CLI.IsInReg)
10984 Attrs.push_back(Attribute::InReg);
10985
10986 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
10987 Attrs);
10988}
10989
10990/// TargetLowering::LowerCallTo - This is the default LowerCallTo
10991/// implementation, which just calls LowerCall.
10992/// FIXME: When all targets are
10993/// migrated to using LowerCall, this hook should be integrated into SDISel.
10994std::pair<SDValue, SDValue>
10996 LLVMContext &Context = CLI.RetTy->getContext();
10997
10998 // Handle the incoming return values from the call.
10999 CLI.Ins.clear();
11000 SmallVector<Type *, 4> RetOrigTys;
11002 auto &DL = CLI.DAG.getDataLayout();
11003 ComputeValueTypes(DL, CLI.OrigRetTy, RetOrigTys, &Offsets);
11004
11005 SmallVector<EVT, 4> RetVTs;
11006 if (CLI.RetTy != CLI.OrigRetTy) {
11007 assert(RetOrigTys.size() == 1 &&
11008 "Only supported for non-aggregate returns");
11009 RetVTs.push_back(getValueType(DL, CLI.RetTy));
11010 } else {
11011 for (Type *Ty : RetOrigTys)
11012 RetVTs.push_back(getValueType(DL, Ty));
11013 }
11014
11015 if (CLI.IsPostTypeLegalization) {
11016 // If we are lowering a libcall after legalization, split the return type.
11017 SmallVector<Type *, 4> OldRetOrigTys;
11018 SmallVector<EVT, 4> OldRetVTs;
11019 SmallVector<TypeSize, 4> OldOffsets;
11020 RetOrigTys.swap(OldRetOrigTys);
11021 RetVTs.swap(OldRetVTs);
11022 Offsets.swap(OldOffsets);
11023
11024 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
11025 EVT RetVT = OldRetVTs[i];
11026 uint64_t Offset = OldOffsets[i];
11027 MVT RegisterVT = getRegisterType(Context, RetVT);
11028 unsigned NumRegs = getNumRegisters(Context, RetVT);
11029 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
11030 RetOrigTys.append(NumRegs, OldRetOrigTys[i]);
11031 RetVTs.append(NumRegs, RegisterVT);
11032 for (unsigned j = 0; j != NumRegs; ++j)
11033 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
11034 }
11035 }
11036
11038 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
11039
11040 bool CanLowerReturn =
11042 CLI.IsVarArg, Outs, Context, CLI.RetTy);
11043
11044 SDValue DemoteStackSlot;
11045 int DemoteStackIdx = -100;
11046 if (!CanLowerReturn) {
11047 // FIXME: equivalent assert?
11048 // assert(!CS.hasInAllocaArgument() &&
11049 // "sret demotion is incompatible with inalloca");
11050 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
11051 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
11053 DemoteStackIdx =
11054 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
11055 Type *StackSlotPtrType = PointerType::get(Context, DL.getAllocaAddrSpace());
11056
11057 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
11058 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11059 Entry.IsSRet = true;
11060 Entry.Alignment = Alignment;
11061 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
11062 CLI.NumFixedArgs += 1;
11063 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11064 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(Context);
11065
11066 // sret demotion isn't compatible with tail-calls, since the sret argument
11067 // points into the callers stack frame.
11068 CLI.IsTailCall = false;
11069 } else {
11070 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11071 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
11072 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
11073 ISD::ArgFlagsTy Flags;
11074 if (NeedsRegBlock) {
11075 Flags.setInConsecutiveRegs();
11076 if (I == RetVTs.size() - 1)
11077 Flags.setInConsecutiveRegsLast();
11078 }
11079 EVT VT = RetVTs[I];
11080 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11081 unsigned NumRegs =
11082 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11083 for (unsigned i = 0; i != NumRegs; ++i) {
11084 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
11086 if (CLI.RetTy->isPointerTy()) {
11087 Ret.Flags.setPointer();
11088 Ret.Flags.setPointerAddrSpace(
11089 cast<PointerType>(CLI.RetTy)->getAddressSpace());
11090 }
11091 if (CLI.RetSExt)
11092 Ret.Flags.setSExt();
11093 if (CLI.RetZExt)
11094 Ret.Flags.setZExt();
11095 if (CLI.IsInReg)
11096 Ret.Flags.setInReg();
11097 CLI.Ins.push_back(Ret);
11098 }
11099 }
11100 }
11101
11102 // We push in swifterror return as the last element of CLI.Ins.
11103 ArgListTy &Args = CLI.getArgs();
11104 if (supportSwiftError()) {
11105 for (const ArgListEntry &Arg : Args) {
11106 if (Arg.IsSwiftError) {
11107 ISD::ArgFlagsTy Flags;
11108 Flags.setSwiftError();
11110 PointerType::getUnqual(Context),
11111 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
11112 CLI.Ins.push_back(Ret);
11113 }
11114 }
11115 }
11116
11117 // Handle all of the outgoing arguments.
11118 CLI.Outs.clear();
11119 CLI.OutVals.clear();
11120 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11121 SmallVector<Type *, 4> OrigArgTys;
11122 ComputeValueTypes(DL, Args[i].OrigTy, OrigArgTys);
11123 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11124 Type *FinalType = Args[i].Ty;
11125 if (Args[i].IsByVal)
11126 FinalType = Args[i].IndirectType;
11127 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11128 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
11129 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
11130 ++Value) {
11131 Type *OrigArgTy = OrigArgTys[Value];
11132 Type *ArgTy = OrigArgTy;
11133 if (Args[i].Ty != Args[i].OrigTy) {
11134 assert(Value == 0 && "Only supported for non-aggregate arguments");
11135 ArgTy = Args[i].Ty;
11136 }
11137
11138 EVT VT = getValueType(DL, ArgTy);
11139 SDValue Op = SDValue(Args[i].Node.getNode(),
11140 Args[i].Node.getResNo() + Value);
11141 ISD::ArgFlagsTy Flags;
11142
11143 // Certain targets (such as MIPS), may have a different ABI alignment
11144 // for a type depending on the context. Give the target a chance to
11145 // specify the alignment it wants.
11146 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11147 Flags.setOrigAlign(OriginalAlignment);
11148
11149 if (i >= CLI.NumFixedArgs)
11150 Flags.setVarArg();
11151 if (ArgTy->isPointerTy()) {
11152 Flags.setPointer();
11153 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
11154 }
11155 if (Args[i].IsZExt)
11156 Flags.setZExt();
11157 if (Args[i].IsSExt)
11158 Flags.setSExt();
11159 if (Args[i].IsNoExt)
11160 Flags.setNoExt();
11161 if (Args[i].IsInReg) {
11162 // If we are using vectorcall calling convention, a structure that is
11163 // passed InReg - is surely an HVA
11165 isa<StructType>(FinalType)) {
11166 // The first value of a structure is marked
11167 if (0 == Value)
11168 Flags.setHvaStart();
11169 Flags.setHva();
11170 }
11171 // Set InReg Flag
11172 Flags.setInReg();
11173 }
11174 if (Args[i].IsSRet)
11175 Flags.setSRet();
11176 if (Args[i].IsSwiftSelf)
11177 Flags.setSwiftSelf();
11178 if (Args[i].IsSwiftAsync)
11179 Flags.setSwiftAsync();
11180 if (Args[i].IsSwiftError)
11181 Flags.setSwiftError();
11182 if (Args[i].IsCFGuardTarget)
11183 Flags.setCFGuardTarget();
11184 if (Args[i].IsByVal)
11185 Flags.setByVal();
11186 if (Args[i].IsByRef)
11187 Flags.setByRef();
11188 if (Args[i].IsPreallocated) {
11189 Flags.setPreallocated();
11190 // Set the byval flag for CCAssignFn callbacks that don't know about
11191 // preallocated. This way we can know how many bytes we should've
11192 // allocated and how many bytes a callee cleanup function will pop. If
11193 // we port preallocated to more targets, we'll have to add custom
11194 // preallocated handling in the various CC lowering callbacks.
11195 Flags.setByVal();
11196 }
11197 if (Args[i].IsInAlloca) {
11198 Flags.setInAlloca();
11199 // Set the byval flag for CCAssignFn callbacks that don't know about
11200 // inalloca. This way we can know how many bytes we should've allocated
11201 // and how many bytes a callee cleanup function will pop. If we port
11202 // inalloca to more targets, we'll have to add custom inalloca handling
11203 // in the various CC lowering callbacks.
11204 Flags.setByVal();
11205 }
11206 Align MemAlign;
11207 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11208 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
11209 Flags.setByValSize(FrameSize);
11210
11211 // info is not there but there are cases it cannot get right.
11212 if (auto MA = Args[i].Alignment)
11213 MemAlign = *MA;
11214 else
11215 MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL);
11216 } else if (auto MA = Args[i].Alignment) {
11217 MemAlign = *MA;
11218 } else {
11219 MemAlign = OriginalAlignment;
11220 }
11221 Flags.setMemAlign(MemAlign);
11222 if (Args[i].IsNest)
11223 Flags.setNest();
11224 if (NeedsRegBlock)
11225 Flags.setInConsecutiveRegs();
11226
11227 MVT PartVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11228 unsigned NumParts =
11229 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11230 SmallVector<SDValue, 4> Parts(NumParts);
11231 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
11232
11233 if (Args[i].IsSExt)
11234 ExtendKind = ISD::SIGN_EXTEND;
11235 else if (Args[i].IsZExt)
11236 ExtendKind = ISD::ZERO_EXTEND;
11237
11238 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11239 // for now.
11240 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11242 assert((CLI.RetTy == Args[i].Ty ||
11243 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11245 Args[i].Ty->getPointerAddressSpace())) &&
11246 RetVTs.size() == NumValues && "unexpected use of 'returned'");
11247 // Before passing 'returned' to the target lowering code, ensure that
11248 // either the register MVT and the actual EVT are the same size or that
11249 // the return value and argument are extended in the same way; in these
11250 // cases it's safe to pass the argument register value unchanged as the
11251 // return register value (although it's at the target's option whether
11252 // to do so)
11253 // TODO: allow code generation to take advantage of partially preserved
11254 // registers rather than clobbering the entire register when the
11255 // parameter extension method is not compatible with the return
11256 // extension method
11257 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11258 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11259 CLI.RetZExt == Args[i].IsZExt))
11260 Flags.setReturned();
11261 }
11262
11263 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
11264 CLI.CallConv, ExtendKind);
11265
11266 for (unsigned j = 0; j != NumParts; ++j) {
11267 // if it isn't first piece, alignment must be 1
11268 // For scalable vectors the scalable part is currently handled
11269 // by individual targets, so we just use the known minimum size here.
11270 ISD::OutputArg MyFlags(
11271 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
11272 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11273 if (NumParts > 1 && j == 0)
11274 MyFlags.Flags.setSplit();
11275 else if (j != 0) {
11276 MyFlags.Flags.setOrigAlign(Align(1));
11277 if (j == NumParts - 1)
11278 MyFlags.Flags.setSplitEnd();
11279 }
11280
11281 CLI.Outs.push_back(MyFlags);
11282 CLI.OutVals.push_back(Parts[j]);
11283 }
11284
11285 if (NeedsRegBlock && Value == NumValues - 1)
11286 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11287 }
11288 }
11289
11291 CLI.Chain = LowerCall(CLI, InVals);
11292
11293 // Update CLI.InVals to use outside of this function.
11294 CLI.InVals = InVals;
11295
11296 // Verify that the target's LowerCall behaved as expected.
11297 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11298 "LowerCall didn't return a valid chain!");
11299 assert((!CLI.IsTailCall || InVals.empty()) &&
11300 "LowerCall emitted a return value for a tail call!");
11301 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11302 "LowerCall didn't emit the correct number of values!");
11303
11304 // For a tail call, the return value is merely live-out and there aren't
11305 // any nodes in the DAG representing it. Return a special value to
11306 // indicate that a tail call has been emitted and no more Instructions
11307 // should be processed in the current block.
11308 if (CLI.IsTailCall) {
11309 CLI.DAG.setRoot(CLI.Chain);
11310 return std::make_pair(SDValue(), SDValue());
11311 }
11312
11313#ifndef NDEBUG
11314 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11315 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11316 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11317 "LowerCall emitted a value with the wrong type!");
11318 }
11319#endif
11320
11321 SmallVector<SDValue, 4> ReturnValues;
11322 if (!CanLowerReturn) {
11323 // The instruction result is the result of loading from the
11324 // hidden sret parameter.
11325 MVT PtrVT = getPointerTy(DL, DL.getAllocaAddrSpace());
11326
11327 unsigned NumValues = RetVTs.size();
11328 ReturnValues.resize(NumValues);
11329 SmallVector<SDValue, 4> Chains(NumValues);
11330
11331 // An aggregate return value cannot wrap around the address space, so
11332 // offsets to its parts don't wrap either.
11334 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
11335 for (unsigned i = 0; i < NumValues; ++i) {
11337 DemoteStackSlot, CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT),
11339 SDValue L = CLI.DAG.getLoad(
11340 RetVTs[i], CLI.DL, CLI.Chain, Add,
11342 DemoteStackIdx, Offsets[i]),
11343 HiddenSRetAlign);
11344 ReturnValues[i] = L;
11345 Chains[i] = L.getValue(1);
11346 }
11347
11348 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
11349 } else {
11350 // Collect the legal value parts into potentially illegal values
11351 // that correspond to the original function's return values.
11352 std::optional<ISD::NodeType> AssertOp;
11353 if (CLI.RetSExt)
11354 AssertOp = ISD::AssertSext;
11355 else if (CLI.RetZExt)
11356 AssertOp = ISD::AssertZext;
11357 unsigned CurReg = 0;
11358 for (EVT VT : RetVTs) {
11359 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11360 unsigned NumRegs =
11361 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11362
11363 ReturnValues.push_back(getCopyFromParts(
11364 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
11365 CLI.Chain, CLI.CallConv, AssertOp));
11366 CurReg += NumRegs;
11367 }
11368
11369 // For a function returning void, there is no return value. We can't create
11370 // such a node, so we just return a null return value in that case. In
11371 // that case, nothing will actually look at the value.
11372 if (ReturnValues.empty())
11373 return std::make_pair(SDValue(), CLI.Chain);
11374 }
11375
11376 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
11377 CLI.DAG.getVTList(RetVTs), ReturnValues);
11378 return std::make_pair(Res, CLI.Chain);
11379}
11380
11381/// Places new result values for the node in Results (their number
11382/// and types must exactly match those of the original return values of
11383/// the node), or leaves Results empty, which indicates that the node is not
11384/// to be custom lowered after all.
11387 SelectionDAG &DAG) const {
11388 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
11389
11390 if (!Res.getNode())
11391 return;
11392
11393 // If the original node has one result, take the return value from
11394 // LowerOperation as is. It might not be result number 0.
11395 if (N->getNumValues() == 1) {
11396 Results.push_back(Res);
11397 return;
11398 }
11399
11400 // If the original node has multiple results, then the return node should
11401 // have the same number of results.
11402 assert((N->getNumValues() == Res->getNumValues()) &&
11403 "Lowering returned the wrong number of results!");
11404
11405 // Places new result values base on N result number.
11406 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11407 Results.push_back(Res.getValue(I));
11408}
11409
11411 llvm_unreachable("LowerOperation not implemented for this target!");
11412}
11413
11415 Register Reg,
11416 ISD::NodeType ExtendType) {
11418 assert((Op.getOpcode() != ISD::CopyFromReg ||
11419 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11420 "Copy from a reg to the same reg!");
11421 assert(!Reg.isPhysical() && "Is a physreg");
11422
11423 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11424 // If this is an InlineAsm we have to match the registers required, not the
11425 // notional registers required by the type.
11426
11427 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11428 std::nullopt); // This is not an ABI copy.
11429 SDValue Chain = DAG.getEntryNode();
11430
11431 if (ExtendType == ISD::ANY_EXTEND) {
11432 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
11433 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11434 ExtendType = PreferredExtendIt->second;
11435 }
11436 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
11437 PendingExports.push_back(Chain);
11438}
11439
11441
11442/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11443/// entry block, return true. This includes arguments used by switches, since
11444/// the switch may expand into multiple basic blocks.
11445static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11446 // With FastISel active, we may be splitting blocks, so force creation
11447 // of virtual registers for all non-dead arguments.
11448 if (FastISel)
11449 return A->use_empty();
11450
11451 const BasicBlock &Entry = A->getParent()->front();
11452 for (const User *U : A->users())
11453 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
11454 return false; // Use not in entry block.
11455
11456 return true;
11457}
11458
11460 DenseMap<const Argument *,
11461 std::pair<const AllocaInst *, const StoreInst *>>;
11462
11463/// Scan the entry block of the function in FuncInfo for arguments that look
11464/// like copies into a local alloca. Record any copied arguments in
11465/// ArgCopyElisionCandidates.
11466static void
11468 FunctionLoweringInfo *FuncInfo,
11469 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
11470 // Record the state of every static alloca used in the entry block. Argument
11471 // allocas are all used in the entry block, so we need approximately as many
11472 // entries as we have arguments.
11473 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
11475 unsigned NumArgs = FuncInfo->Fn->arg_size();
11476 StaticAllocas.reserve(NumArgs * 2);
11477
11478 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
11479 if (!V)
11480 return nullptr;
11481 V = V->stripPointerCasts();
11482 const auto *AI = dyn_cast<AllocaInst>(V);
11483 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
11484 return nullptr;
11485 auto Iter = StaticAllocas.insert({AI, Unknown});
11486 return &Iter.first->second;
11487 };
11488
11489 // Look for stores of arguments to static allocas. Look through bitcasts and
11490 // GEPs to handle type coercions, as long as the alloca is fully initialized
11491 // by the store. Any non-store use of an alloca escapes it and any subsequent
11492 // unanalyzed store might write it.
11493 // FIXME: Handle structs initialized with multiple stores.
11494 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11495 // Look for stores, and handle non-store uses conservatively.
11496 const auto *SI = dyn_cast<StoreInst>(&I);
11497 if (!SI) {
11498 // We will look through cast uses, so ignore them completely.
11499 if (I.isCast())
11500 continue;
11501 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11502 // to allocas.
11503 if (I.isDebugOrPseudoInst())
11504 continue;
11505 // This is an unknown instruction. Assume it escapes or writes to all
11506 // static alloca operands.
11507 for (const Use &U : I.operands()) {
11508 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11509 *Info = StaticAllocaInfo::Clobbered;
11510 }
11511 continue;
11512 }
11513
11514 // If the stored value is a static alloca, mark it as escaped.
11515 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11516 *Info = StaticAllocaInfo::Clobbered;
11517
11518 // Check if the destination is a static alloca.
11519 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11520 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11521 if (!Info)
11522 continue;
11523 const AllocaInst *AI = cast<AllocaInst>(Dst);
11524
11525 // Skip allocas that have been initialized or clobbered.
11526 if (*Info != StaticAllocaInfo::Unknown)
11527 continue;
11528
11529 // Check if the stored value is an argument, and that this store fully
11530 // initializes the alloca.
11531 // If the argument type has padding bits we can't directly forward a pointer
11532 // as the upper bits may contain garbage.
11533 // Don't elide copies from the same argument twice.
11534 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11535 const auto *Arg = dyn_cast<Argument>(Val);
11536 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11537 Arg->getType()->isEmptyTy() ||
11538 DL.getTypeStoreSize(Arg->getType()) !=
11539 DL.getTypeAllocSize(AI->getAllocatedType()) ||
11540 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
11541 ArgCopyElisionCandidates.count(Arg)) {
11542 *Info = StaticAllocaInfo::Clobbered;
11543 continue;
11544 }
11545
11546 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11547 << '\n');
11548
11549 // Mark this alloca and store for argument copy elision.
11550 *Info = StaticAllocaInfo::Elidable;
11551 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
11552
11553 // Stop scanning if we've seen all arguments. This will happen early in -O0
11554 // builds, which is useful, because -O0 builds have large entry blocks and
11555 // many allocas.
11556 if (ArgCopyElisionCandidates.size() == NumArgs)
11557 break;
11558 }
11559}
11560
11561/// Try to elide argument copies from memory into a local alloca. Succeeds if
11562/// ArgVal is a load from a suitable fixed stack object.
11565 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11566 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11567 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11568 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11569 // Check if this is a load from a fixed stack object.
11570 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
11571 if (!LNode)
11572 return;
11573 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
11574 if (!FINode)
11575 return;
11576
11577 // Check that the fixed stack object is the right size and alignment.
11578 // Look at the alignment that the user wrote on the alloca instead of looking
11579 // at the stack object.
11580 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
11581 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11582 const AllocaInst *AI = ArgCopyIter->second.first;
11583 int FixedIndex = FINode->getIndex();
11584 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11585 int OldIndex = AllocaIndex;
11586 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11587 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
11588 LLVM_DEBUG(
11589 dbgs() << " argument copy elision failed due to bad fixed stack "
11590 "object size\n");
11591 return;
11592 }
11593 Align RequiredAlignment = AI->getAlign();
11594 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
11595 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11596 "greater than stack argument alignment ("
11597 << DebugStr(RequiredAlignment) << " vs "
11598 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11599 return;
11600 }
11601
11602 // Perform the elision. Delete the old stack object and replace its only use
11603 // in the variable info map. Mark the stack object as mutable and aliased.
11604 LLVM_DEBUG({
11605 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11606 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11607 << '\n';
11608 });
11609 MFI.RemoveStackObject(OldIndex);
11610 MFI.setIsImmutableObjectIndex(FixedIndex, false);
11611 MFI.setIsAliasedObjectIndex(FixedIndex, true);
11612 AllocaIndex = FixedIndex;
11613 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
11614 for (SDValue ArgVal : ArgVals)
11615 Chains.push_back(ArgVal.getValue(1));
11616
11617 // Avoid emitting code for the store implementing the copy.
11618 const StoreInst *SI = ArgCopyIter->second.second;
11619 ElidedArgCopyInstrs.insert(SI);
11620
11621 // Check for uses of the argument again so that we can avoid exporting ArgVal
11622 // if it is't used by anything other than the store.
11623 for (const Value *U : Arg.users()) {
11624 if (U != SI) {
11625 ArgHasUses = true;
11626 break;
11627 }
11628 }
11629}
11630
11631void SelectionDAGISel::LowerArguments(const Function &F) {
11632 SelectionDAG &DAG = SDB->DAG;
11633 SDLoc dl = SDB->getCurSDLoc();
11634 const DataLayout &DL = DAG.getDataLayout();
11636
11637 // In Naked functions we aren't going to save any registers.
11638 if (F.hasFnAttribute(Attribute::Naked))
11639 return;
11640
11641 if (!FuncInfo->CanLowerReturn) {
11642 // Put in an sret pointer parameter before all the other parameters.
11643 MVT ValueVT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
11644
11645 ISD::ArgFlagsTy Flags;
11646 Flags.setSRet();
11647 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVT);
11648 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
11650 Ins.push_back(RetArg);
11651 }
11652
11653 // Look for stores of arguments to static allocas. Mark such arguments with a
11654 // flag to ask the target to give us the memory location of that argument if
11655 // available.
11656 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11658 ArgCopyElisionCandidates);
11659
11660 // Set up the incoming argument description vector.
11661 for (const Argument &Arg : F.args()) {
11662 unsigned ArgNo = Arg.getArgNo();
11664 ComputeValueTypes(DAG.getDataLayout(), Arg.getType(), Types);
11665 bool isArgValueUsed = !Arg.use_empty();
11666 unsigned PartBase = 0;
11667 Type *FinalType = Arg.getType();
11668 if (Arg.hasAttribute(Attribute::ByVal))
11669 FinalType = Arg.getParamByValType();
11670 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11671 FinalType, F.getCallingConv(), F.isVarArg(), DL);
11672 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
11673 ++Value) {
11674 Type *ArgTy = Types[Value];
11675 EVT VT = TLI->getValueType(DL, ArgTy);
11676 ISD::ArgFlagsTy Flags;
11677
11678 if (ArgTy->isPointerTy()) {
11679 Flags.setPointer();
11680 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
11681 }
11682 if (Arg.hasAttribute(Attribute::ZExt))
11683 Flags.setZExt();
11684 if (Arg.hasAttribute(Attribute::SExt))
11685 Flags.setSExt();
11686 if (Arg.hasAttribute(Attribute::InReg)) {
11687 // If we are using vectorcall calling convention, a structure that is
11688 // passed InReg - is surely an HVA
11689 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11690 isa<StructType>(Arg.getType())) {
11691 // The first value of a structure is marked
11692 if (0 == Value)
11693 Flags.setHvaStart();
11694 Flags.setHva();
11695 }
11696 // Set InReg Flag
11697 Flags.setInReg();
11698 }
11699 if (Arg.hasAttribute(Attribute::StructRet))
11700 Flags.setSRet();
11701 if (Arg.hasAttribute(Attribute::SwiftSelf))
11702 Flags.setSwiftSelf();
11703 if (Arg.hasAttribute(Attribute::SwiftAsync))
11704 Flags.setSwiftAsync();
11705 if (Arg.hasAttribute(Attribute::SwiftError))
11706 Flags.setSwiftError();
11707 if (Arg.hasAttribute(Attribute::ByVal))
11708 Flags.setByVal();
11709 if (Arg.hasAttribute(Attribute::ByRef))
11710 Flags.setByRef();
11711 if (Arg.hasAttribute(Attribute::InAlloca)) {
11712 Flags.setInAlloca();
11713 // Set the byval flag for CCAssignFn callbacks that don't know about
11714 // inalloca. This way we can know how many bytes we should've allocated
11715 // and how many bytes a callee cleanup function will pop. If we port
11716 // inalloca to more targets, we'll have to add custom inalloca handling
11717 // in the various CC lowering callbacks.
11718 Flags.setByVal();
11719 }
11720 if (Arg.hasAttribute(Attribute::Preallocated)) {
11721 Flags.setPreallocated();
11722 // Set the byval flag for CCAssignFn callbacks that don't know about
11723 // preallocated. This way we can know how many bytes we should've
11724 // allocated and how many bytes a callee cleanup function will pop. If
11725 // we port preallocated to more targets, we'll have to add custom
11726 // preallocated handling in the various CC lowering callbacks.
11727 Flags.setByVal();
11728 }
11729
11730 // Certain targets (such as MIPS), may have a different ABI alignment
11731 // for a type depending on the context. Give the target a chance to
11732 // specify the alignment it wants.
11733 const Align OriginalAlignment(
11734 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
11735 Flags.setOrigAlign(OriginalAlignment);
11736
11737 Align MemAlign;
11738 Type *ArgMemTy = nullptr;
11739 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11740 Flags.isByRef()) {
11741 if (!ArgMemTy)
11742 ArgMemTy = Arg.getPointeeInMemoryValueType();
11743
11744 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
11745
11746 // For in-memory arguments, size and alignment should be passed from FE.
11747 // BE will guess if this info is not there but there are cases it cannot
11748 // get right.
11749 if (auto ParamAlign = Arg.getParamStackAlign())
11750 MemAlign = *ParamAlign;
11751 else if ((ParamAlign = Arg.getParamAlign()))
11752 MemAlign = *ParamAlign;
11753 else
11754 MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL);
11755 if (Flags.isByRef())
11756 Flags.setByRefSize(MemSize);
11757 else
11758 Flags.setByValSize(MemSize);
11759 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
11760 MemAlign = *ParamAlign;
11761 } else {
11762 MemAlign = OriginalAlignment;
11763 }
11764 Flags.setMemAlign(MemAlign);
11765
11766 if (Arg.hasAttribute(Attribute::Nest))
11767 Flags.setNest();
11768 if (NeedsRegBlock)
11769 Flags.setInConsecutiveRegs();
11770 if (ArgCopyElisionCandidates.count(&Arg))
11771 Flags.setCopyElisionCandidate();
11772 if (Arg.hasAttribute(Attribute::Returned))
11773 Flags.setReturned();
11774
11775 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
11776 *CurDAG->getContext(), F.getCallingConv(), VT);
11777 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
11778 *CurDAG->getContext(), F.getCallingConv(), VT);
11779 for (unsigned i = 0; i != NumRegs; ++i) {
11780 // For scalable vectors, use the minimum size; individual targets
11781 // are responsible for handling scalable vector arguments and
11782 // return values.
11783 ISD::InputArg MyFlags(
11784 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
11785 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
11786 if (NumRegs > 1 && i == 0)
11787 MyFlags.Flags.setSplit();
11788 // if it isn't first piece, alignment must be 1
11789 else if (i > 0) {
11790 MyFlags.Flags.setOrigAlign(Align(1));
11791 if (i == NumRegs - 1)
11792 MyFlags.Flags.setSplitEnd();
11793 }
11794 Ins.push_back(MyFlags);
11795 }
11796 if (NeedsRegBlock && Value == NumValues - 1)
11797 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11798 PartBase += VT.getStoreSize().getKnownMinValue();
11799 }
11800 }
11801
11802 // Call the target to set up the argument values.
11804 SDValue NewRoot = TLI->LowerFormalArguments(
11805 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
11806
11807 // Verify that the target's LowerFormalArguments behaved as expected.
11808 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
11809 "LowerFormalArguments didn't return a valid chain!");
11810 assert(InVals.size() == Ins.size() &&
11811 "LowerFormalArguments didn't emit the correct number of values!");
11812 LLVM_DEBUG({
11813 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
11814 assert(InVals[i].getNode() &&
11815 "LowerFormalArguments emitted a null value!");
11816 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11817 "LowerFormalArguments emitted a value with the wrong type!");
11818 }
11819 });
11820
11821 // Update the DAG with the new chain value resulting from argument lowering.
11822 DAG.setRoot(NewRoot);
11823
11824 // Set up the argument values.
11825 unsigned i = 0;
11826 if (!FuncInfo->CanLowerReturn) {
11827 // Create a virtual register for the sret pointer, and put in a copy
11828 // from the sret argument into it.
11829 MVT VT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
11830 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
11831 std::optional<ISD::NodeType> AssertOp;
11832 SDValue ArgValue =
11833 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
11834 F.getCallingConv(), AssertOp);
11835
11836 MachineFunction& MF = SDB->DAG.getMachineFunction();
11837 MachineRegisterInfo& RegInfo = MF.getRegInfo();
11838 Register SRetReg =
11839 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
11840 FuncInfo->DemoteRegister = SRetReg;
11841 NewRoot =
11842 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
11843 DAG.setRoot(NewRoot);
11844
11845 // i indexes lowered arguments. Bump it past the hidden sret argument.
11846 ++i;
11847 }
11848
11850 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
11851 for (const Argument &Arg : F.args()) {
11852 SmallVector<SDValue, 4> ArgValues;
11853 SmallVector<EVT, 4> ValueVTs;
11854 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11855 unsigned NumValues = ValueVTs.size();
11856 if (NumValues == 0)
11857 continue;
11858
11859 bool ArgHasUses = !Arg.use_empty();
11860
11861 // Elide the copying store if the target loaded this argument from a
11862 // suitable fixed stack object.
11863 if (Ins[i].Flags.isCopyElisionCandidate()) {
11864 unsigned NumParts = 0;
11865 for (EVT VT : ValueVTs)
11866 NumParts += TLI->getNumRegistersForCallingConv(*CurDAG->getContext(),
11867 F.getCallingConv(), VT);
11868
11869 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
11870 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
11871 ArrayRef(&InVals[i], NumParts), ArgHasUses);
11872 }
11873
11874 // If this argument is unused then remember its value. It is used to generate
11875 // debugging information.
11876 bool isSwiftErrorArg =
11877 TLI->supportSwiftError() &&
11878 Arg.hasAttribute(Attribute::SwiftError);
11879 if (!ArgHasUses && !isSwiftErrorArg) {
11880 SDB->setUnusedArgValue(&Arg, InVals[i]);
11881
11882 // Also remember any frame index for use in FastISel.
11883 if (FrameIndexSDNode *FI =
11885 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11886 }
11887
11888 for (unsigned Val = 0; Val != NumValues; ++Val) {
11889 EVT VT = ValueVTs[Val];
11890 MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
11891 F.getCallingConv(), VT);
11892 unsigned NumParts = TLI->getNumRegistersForCallingConv(
11893 *CurDAG->getContext(), F.getCallingConv(), VT);
11894
11895 // Even an apparent 'unused' swifterror argument needs to be returned. So
11896 // we do generate a copy for it that can be used on return from the
11897 // function.
11898 if (ArgHasUses || isSwiftErrorArg) {
11899 std::optional<ISD::NodeType> AssertOp;
11900 if (Arg.hasAttribute(Attribute::SExt))
11901 AssertOp = ISD::AssertSext;
11902 else if (Arg.hasAttribute(Attribute::ZExt))
11903 AssertOp = ISD::AssertZext;
11904
11905 SDValue OutVal =
11906 getCopyFromParts(DAG, dl, &InVals[i], NumParts, PartVT, VT, nullptr,
11907 NewRoot, F.getCallingConv(), AssertOp);
11908
11909 FPClassTest NoFPClass = Arg.getNoFPClass();
11910 if (NoFPClass != fcNone) {
11911 SDValue SDNoFPClass = DAG.getTargetConstant(
11912 static_cast<uint64_t>(NoFPClass), dl, MVT::i32);
11913 OutVal = DAG.getNode(ISD::AssertNoFPClass, dl, OutVal.getValueType(),
11914 OutVal, SDNoFPClass);
11915 }
11916 ArgValues.push_back(OutVal);
11917 }
11918
11919 i += NumParts;
11920 }
11921
11922 // We don't need to do anything else for unused arguments.
11923 if (ArgValues.empty())
11924 continue;
11925
11926 // Note down frame index.
11927 if (FrameIndexSDNode *FI =
11928 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
11929 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11930
11931 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
11932 SDB->getCurSDLoc());
11933
11934 SDB->setValue(&Arg, Res);
11935 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
11936 // We want to associate the argument with the frame index, among
11937 // involved operands, that correspond to the lowest address. The
11938 // getCopyFromParts function, called earlier, is swapping the order of
11939 // the operands to BUILD_PAIR depending on endianness. The result of
11940 // that swapping is that the least significant bits of the argument will
11941 // be in the first operand of the BUILD_PAIR node, and the most
11942 // significant bits will be in the second operand.
11943 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
11944 if (LoadSDNode *LNode =
11945 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
11946 if (FrameIndexSDNode *FI =
11947 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
11948 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11949 }
11950
11951 // Analyses past this point are naive and don't expect an assertion.
11952 if (Res.getOpcode() == ISD::AssertZext)
11953 Res = Res.getOperand(0);
11954
11955 // Update the SwiftErrorVRegDefMap.
11956 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
11957 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11958 if (Reg.isVirtual())
11959 SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
11960 Reg);
11961 }
11962
11963 // If this argument is live outside of the entry block, insert a copy from
11964 // wherever we got it to the vreg that other BB's will reference it as.
11965 if (Res.getOpcode() == ISD::CopyFromReg) {
11966 // If we can, though, try to skip creating an unnecessary vreg.
11967 // FIXME: This isn't very clean... it would be nice to make this more
11968 // general.
11969 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11970 if (Reg.isVirtual()) {
11971 FuncInfo->ValueMap[&Arg] = Reg;
11972 continue;
11973 }
11974 }
11975 if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
11976 FuncInfo->InitializeRegForValue(&Arg);
11977 SDB->CopyToExportRegsIfNeeded(&Arg);
11978 }
11979 }
11980
11981 if (!Chains.empty()) {
11982 Chains.push_back(NewRoot);
11983 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
11984 }
11985
11986 DAG.setRoot(NewRoot);
11987
11988 assert(i == InVals.size() && "Argument register count mismatch!");
11989
11990 // If any argument copy elisions occurred and we have debug info, update the
11991 // stale frame indices used in the dbg.declare variable info table.
11992 if (!ArgCopyElisionFrameIndexMap.empty()) {
11993 for (MachineFunction::VariableDbgInfo &VI :
11994 MF->getInStackSlotVariableDbgInfo()) {
11995 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
11996 if (I != ArgCopyElisionFrameIndexMap.end())
11997 VI.updateStackSlot(I->second);
11998 }
11999 }
12000
12001 // Finally, if the target has anything special to do, allow it to do so.
12003}
12004
12005/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
12006/// ensure constants are generated when needed. Remember the virtual registers
12007/// that need to be added to the Machine PHI nodes as input. We cannot just
12008/// directly add them, because expansion might result in multiple MBB's for one
12009/// BB. As such, the start of the BB might correspond to a different MBB than
12010/// the end.
12011void
12012SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
12013 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12014
12015 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12016
12017 // Check PHI nodes in successors that expect a value to be available from this
12018 // block.
12019 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
12020 if (!isa<PHINode>(SuccBB->begin())) continue;
12021 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB);
12022
12023 // If this terminator has multiple identical successors (common for
12024 // switches), only handle each succ once.
12025 if (!SuccsHandled.insert(SuccMBB).second)
12026 continue;
12027
12029
12030 // At this point we know that there is a 1-1 correspondence between LLVM PHI
12031 // nodes and Machine PHI nodes, but the incoming operands have not been
12032 // emitted yet.
12033 for (const PHINode &PN : SuccBB->phis()) {
12034 // Ignore dead phi's.
12035 if (PN.use_empty())
12036 continue;
12037
12038 // Skip empty types
12039 if (PN.getType()->isEmptyTy())
12040 continue;
12041
12042 Register Reg;
12043 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
12044
12045 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
12046 Register &RegOut = ConstantsOut[C];
12047 if (!RegOut) {
12048 RegOut = FuncInfo.CreateRegs(&PN);
12049 // We need to zero/sign extend ConstantInt phi operands to match
12050 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12051 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12052 if (auto *CI = dyn_cast<ConstantInt>(C))
12053 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
12055 CopyValueToVirtualRegister(C, RegOut, ExtendType);
12056 }
12057 Reg = RegOut;
12058 } else {
12060 FuncInfo.ValueMap.find(PHIOp);
12061 if (I != FuncInfo.ValueMap.end())
12062 Reg = I->second;
12063 else {
12064 assert(isa<AllocaInst>(PHIOp) &&
12065 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12066 "Didn't codegen value into a register!??");
12067 Reg = FuncInfo.CreateRegs(&PN);
12069 }
12070 }
12071
12072 // Remember that this register needs to added to the machine PHI node as
12073 // the input for this MBB.
12074 SmallVector<EVT, 4> ValueVTs;
12075 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
12076 for (EVT VT : ValueVTs) {
12077 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
12078 for (unsigned i = 0; i != NumRegisters; ++i)
12079 FuncInfo.PHINodesToUpdate.emplace_back(&*MBBI++, Reg + i);
12080 Reg += NumRegisters;
12081 }
12082 }
12083 }
12084
12085 ConstantsOut.clear();
12086}
12087
12088MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12090 if (++I == FuncInfo.MF->end())
12091 return nullptr;
12092 return &*I;
12093}
12094
12095/// During lowering new call nodes can be created (such as memset, etc.).
12096/// Those will become new roots of the current DAG, but complications arise
12097/// when they are tail calls. In such cases, the call lowering will update
12098/// the root, but the builder still needs to know that a tail call has been
12099/// lowered in order to avoid generating an additional return.
12100void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12101 // If the node is null, we do have a tail call.
12102 if (MaybeTC.getNode() != nullptr)
12103 DAG.setRoot(MaybeTC);
12104 else
12105 HasTailCall = true;
12106}
12107
12108void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12109 MachineBasicBlock *SwitchMBB,
12110 MachineBasicBlock *DefaultMBB) {
12111 MachineFunction *CurMF = FuncInfo.MF;
12112 MachineBasicBlock *NextMBB = nullptr;
12114 if (++BBI != FuncInfo.MF->end())
12115 NextMBB = &*BBI;
12116
12117 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12118
12119 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12120
12121 if (Size == 2 && W.MBB == SwitchMBB) {
12122 // If any two of the cases has the same destination, and if one value
12123 // is the same as the other, but has one bit unset that the other has set,
12124 // use bit manipulation to do two compares at once. For example:
12125 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12126 // TODO: This could be extended to merge any 2 cases in switches with 3
12127 // cases.
12128 // TODO: Handle cases where W.CaseBB != SwitchBB.
12129 CaseCluster &Small = *W.FirstCluster;
12130 CaseCluster &Big = *W.LastCluster;
12131
12132 if (Small.Low == Small.High && Big.Low == Big.High &&
12133 Small.MBB == Big.MBB) {
12134 const APInt &SmallValue = Small.Low->getValue();
12135 const APInt &BigValue = Big.Low->getValue();
12136
12137 // Check that there is only one bit different.
12138 APInt CommonBit = BigValue ^ SmallValue;
12139 if (CommonBit.isPowerOf2()) {
12140 SDValue CondLHS = getValue(Cond);
12141 EVT VT = CondLHS.getValueType();
12142 SDLoc DL = getCurSDLoc();
12143
12144 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
12145 DAG.getConstant(CommonBit, DL, VT));
12146 SDValue Cond = DAG.getSetCC(
12147 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
12148 ISD::SETEQ);
12149
12150 // Update successor info.
12151 // Both Small and Big will jump to Small.BB, so we sum up the
12152 // probabilities.
12153 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
12154 if (BPI)
12155 addSuccessorWithProb(
12156 SwitchMBB, DefaultMBB,
12157 // The default destination is the first successor in IR.
12158 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
12159 else
12160 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12161
12162 // Insert the true branch.
12163 SDValue BrCond =
12164 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
12165 DAG.getBasicBlock(Small.MBB));
12166 // Insert the false branch.
12167 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
12168 DAG.getBasicBlock(DefaultMBB));
12169
12170 DAG.setRoot(BrCond);
12171 return;
12172 }
12173 }
12174 }
12175
12176 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12177 // Here, we order cases by probability so the most likely case will be
12178 // checked first. However, two clusters can have the same probability in
12179 // which case their relative ordering is non-deterministic. So we use Low
12180 // as a tie-breaker as clusters are guaranteed to never overlap.
12181 llvm::sort(W.FirstCluster, W.LastCluster + 1,
12182 [](const CaseCluster &a, const CaseCluster &b) {
12183 return a.Prob != b.Prob ?
12184 a.Prob > b.Prob :
12185 a.Low->getValue().slt(b.Low->getValue());
12186 });
12187
12188 // Rearrange the case blocks so that the last one falls through if possible
12189 // without changing the order of probabilities.
12190 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12191 --I;
12192 if (I->Prob > W.LastCluster->Prob)
12193 break;
12194 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12195 std::swap(*I, *W.LastCluster);
12196 break;
12197 }
12198 }
12199 }
12200
12201 // Compute total probability.
12202 BranchProbability DefaultProb = W.DefaultProb;
12203 BranchProbability UnhandledProbs = DefaultProb;
12204 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12205 UnhandledProbs += I->Prob;
12206
12207 MachineBasicBlock *CurMBB = W.MBB;
12208 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12209 bool FallthroughUnreachable = false;
12210 MachineBasicBlock *Fallthrough;
12211 if (I == W.LastCluster) {
12212 // For the last cluster, fall through to the default destination.
12213 Fallthrough = DefaultMBB;
12214 FallthroughUnreachable = isa<UnreachableInst>(
12215 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
12216 } else {
12217 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
12218 CurMF->insert(BBI, Fallthrough);
12219 // Put Cond in a virtual register to make it available from the new blocks.
12221 }
12222 UnhandledProbs -= I->Prob;
12223
12224 switch (I->Kind) {
12225 case CC_JumpTable: {
12226 // FIXME: Optimize away range check based on pivot comparisons.
12227 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12228 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12229
12230 // The jump block hasn't been inserted yet; insert it here.
12231 MachineBasicBlock *JumpMBB = JT->MBB;
12232 CurMF->insert(BBI, JumpMBB);
12233
12234 auto JumpProb = I->Prob;
12235 auto FallthroughProb = UnhandledProbs;
12236
12237 // If the default statement is a target of the jump table, we evenly
12238 // distribute the default probability to successors of CurMBB. Also
12239 // update the probability on the edge from JumpMBB to Fallthrough.
12240 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12241 SE = JumpMBB->succ_end();
12242 SI != SE; ++SI) {
12243 if (*SI == DefaultMBB) {
12244 JumpProb += DefaultProb / 2;
12245 FallthroughProb -= DefaultProb / 2;
12246 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
12247 JumpMBB->normalizeSuccProbs();
12248 break;
12249 }
12250 }
12251
12252 // If the default clause is unreachable, propagate that knowledge into
12253 // JTH->FallthroughUnreachable which will use it to suppress the range
12254 // check.
12255 //
12256 // However, don't do this if we're doing branch target enforcement,
12257 // because a table branch _without_ a range check can be a tempting JOP
12258 // gadget - out-of-bounds inputs that are impossible in correct
12259 // execution become possible again if an attacker can influence the
12260 // control flow. So if an attacker doesn't already have a BTI bypass
12261 // available, we don't want them to be able to get one out of this
12262 // table branch.
12263 if (FallthroughUnreachable) {
12264 Function &CurFunc = CurMF->getFunction();
12265 if (!CurFunc.hasFnAttribute("branch-target-enforcement"))
12266 JTH->FallthroughUnreachable = true;
12267 }
12268
12269 if (!JTH->FallthroughUnreachable)
12270 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12271 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12272 CurMBB->normalizeSuccProbs();
12273
12274 // The jump table header will be inserted in our current block, do the
12275 // range check, and fall through to our fallthrough block.
12276 JTH->HeaderBB = CurMBB;
12277 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12278
12279 // If we're in the right place, emit the jump table header right now.
12280 if (CurMBB == SwitchMBB) {
12281 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
12282 JTH->Emitted = true;
12283 }
12284 break;
12285 }
12286 case CC_BitTests: {
12287 // FIXME: Optimize away range check based on pivot comparisons.
12288 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12289
12290 // The bit test blocks haven't been inserted yet; insert them here.
12291 for (BitTestCase &BTC : BTB->Cases)
12292 CurMF->insert(BBI, BTC.ThisBB);
12293
12294 // Fill in fields of the BitTestBlock.
12295 BTB->Parent = CurMBB;
12296 BTB->Default = Fallthrough;
12297
12298 BTB->DefaultProb = UnhandledProbs;
12299 // If the cases in bit test don't form a contiguous range, we evenly
12300 // distribute the probability on the edge to Fallthrough to two
12301 // successors of CurMBB.
12302 if (!BTB->ContiguousRange) {
12303 BTB->Prob += DefaultProb / 2;
12304 BTB->DefaultProb -= DefaultProb / 2;
12305 }
12306
12307 if (FallthroughUnreachable)
12308 BTB->FallthroughUnreachable = true;
12309
12310 // If we're in the right place, emit the bit test header right now.
12311 if (CurMBB == SwitchMBB) {
12312 visitBitTestHeader(*BTB, SwitchMBB);
12313 BTB->Emitted = true;
12314 }
12315 break;
12316 }
12317 case CC_Range: {
12318 const Value *RHS, *LHS, *MHS;
12319 ISD::CondCode CC;
12320 if (I->Low == I->High) {
12321 // Check Cond == I->Low.
12322 CC = ISD::SETEQ;
12323 LHS = Cond;
12324 RHS=I->Low;
12325 MHS = nullptr;
12326 } else {
12327 // Check I->Low <= Cond <= I->High.
12328 CC = ISD::SETLE;
12329 LHS = I->Low;
12330 MHS = Cond;
12331 RHS = I->High;
12332 }
12333
12334 // If Fallthrough is unreachable, fold away the comparison.
12335 if (FallthroughUnreachable)
12336 CC = ISD::SETTRUE;
12337
12338 // The false probability is the sum of all unhandled cases.
12339 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12340 getCurSDLoc(), I->Prob, UnhandledProbs);
12341
12342 if (CurMBB == SwitchMBB)
12343 visitSwitchCase(CB, SwitchMBB);
12344 else
12345 SL->SwitchCases.push_back(CB);
12346
12347 break;
12348 }
12349 }
12350 CurMBB = Fallthrough;
12351 }
12352}
12353
12354void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12355 const SwitchWorkListItem &W,
12356 Value *Cond,
12357 MachineBasicBlock *SwitchMBB) {
12358 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12359 "Clusters not sorted?");
12360 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12361
12362 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12363 SL->computeSplitWorkItemInfo(W);
12364
12365 // Use the first element on the right as pivot since we will make less-than
12366 // comparisons against it.
12367 CaseClusterIt PivotCluster = FirstRight;
12368 assert(PivotCluster > W.FirstCluster);
12369 assert(PivotCluster <= W.LastCluster);
12370
12371 CaseClusterIt FirstLeft = W.FirstCluster;
12372 CaseClusterIt LastRight = W.LastCluster;
12373
12374 const ConstantInt *Pivot = PivotCluster->Low;
12375
12376 // New blocks will be inserted immediately after the current one.
12378 ++BBI;
12379
12380 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12381 // we can branch to its destination directly if it's squeezed exactly in
12382 // between the known lower bound and Pivot - 1.
12383 MachineBasicBlock *LeftMBB;
12384 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12385 FirstLeft->Low == W.GE &&
12386 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12387 LeftMBB = FirstLeft->MBB;
12388 } else {
12389 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12390 FuncInfo.MF->insert(BBI, LeftMBB);
12391 WorkList.push_back(
12392 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
12393 // Put Cond in a virtual register to make it available from the new blocks.
12395 }
12396
12397 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12398 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12399 // directly if RHS.High equals the current upper bound.
12400 MachineBasicBlock *RightMBB;
12401 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12402 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12403 RightMBB = FirstRight->MBB;
12404 } else {
12405 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12406 FuncInfo.MF->insert(BBI, RightMBB);
12407 WorkList.push_back(
12408 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
12409 // Put Cond in a virtual register to make it available from the new blocks.
12411 }
12412
12413 // Create the CaseBlock record that will be used to lower the branch.
12414 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12415 getCurSDLoc(), LeftProb, RightProb);
12416
12417 if (W.MBB == SwitchMBB)
12418 visitSwitchCase(CB, SwitchMBB);
12419 else
12420 SL->SwitchCases.push_back(CB);
12421}
12422
12423// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12424// from the swith statement.
12426 BranchProbability PeeledCaseProb) {
12427 if (PeeledCaseProb == BranchProbability::getOne())
12429 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12430
12431 uint32_t Numerator = CaseProb.getNumerator();
12432 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
12433 return BranchProbability(Numerator, std::max(Numerator, Denominator));
12434}
12435
12436// Try to peel the top probability case if it exceeds the threshold.
12437// Return current MachineBasicBlock for the switch statement if the peeling
12438// does not occur.
12439// If the peeling is performed, return the newly created MachineBasicBlock
12440// for the peeled switch statement. Also update Clusters to remove the peeled
12441// case. PeeledCaseProb is the BranchProbability for the peeled case.
12442MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12443 const SwitchInst &SI, CaseClusterVector &Clusters,
12444 BranchProbability &PeeledCaseProb) {
12445 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12446 // Don't perform if there is only one cluster or optimizing for size.
12447 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12448 TM.getOptLevel() == CodeGenOptLevel::None ||
12449 SwitchMBB->getParent()->getFunction().hasMinSize())
12450 return SwitchMBB;
12451
12452 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
12453 unsigned PeeledCaseIndex = 0;
12454 bool SwitchPeeled = false;
12455 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12456 CaseCluster &CC = Clusters[Index];
12457 if (CC.Prob < TopCaseProb)
12458 continue;
12459 TopCaseProb = CC.Prob;
12460 PeeledCaseIndex = Index;
12461 SwitchPeeled = true;
12462 }
12463 if (!SwitchPeeled)
12464 return SwitchMBB;
12465
12466 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12467 << TopCaseProb << "\n");
12468
12469 // Record the MBB for the peeled switch statement.
12470 MachineFunction::iterator BBI(SwitchMBB);
12471 ++BBI;
12472 MachineBasicBlock *PeeledSwitchMBB =
12473 FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
12474 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
12475
12476 ExportFromCurrentBlock(SI.getCondition());
12477 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12478 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
12479 nullptr, nullptr, TopCaseProb.getCompl()};
12480 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
12481
12482 Clusters.erase(PeeledCaseIt);
12483 for (CaseCluster &CC : Clusters) {
12484 LLVM_DEBUG(
12485 dbgs() << "Scale the probablity for one cluster, before scaling: "
12486 << CC.Prob << "\n");
12487 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
12488 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12489 }
12490 PeeledCaseProb = TopCaseProb;
12491 return PeeledSwitchMBB;
12492}
12493
12494void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12495 // Extract cases from the switch.
12496 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12497 CaseClusterVector Clusters;
12498 Clusters.reserve(SI.getNumCases());
12499 for (auto I : SI.cases()) {
12500 MachineBasicBlock *Succ = FuncInfo.getMBB(I.getCaseSuccessor());
12501 const ConstantInt *CaseVal = I.getCaseValue();
12502 BranchProbability Prob =
12503 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
12504 : BranchProbability(1, SI.getNumCases() + 1);
12505 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
12506 }
12507
12508 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(SI.getDefaultDest());
12509
12510 // Cluster adjacent cases with the same destination. We do this at all
12511 // optimization levels because it's cheap to do and will make codegen faster
12512 // if there are many clusters.
12513 sortAndRangeify(Clusters);
12514
12515 // The branch probablity of the peeled case.
12516 BranchProbability PeeledCaseProb = BranchProbability::getZero();
12517 MachineBasicBlock *PeeledSwitchMBB =
12518 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12519
12520 // If there is only the default destination, jump there directly.
12521 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12522 if (Clusters.empty()) {
12523 assert(PeeledSwitchMBB == SwitchMBB);
12524 SwitchMBB->addSuccessor(DefaultMBB);
12525 if (DefaultMBB != NextBlock(SwitchMBB)) {
12526 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
12527 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
12528 }
12529 return;
12530 }
12531
12532 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
12533 DAG.getBFI());
12534 SL->findBitTestClusters(Clusters, &SI);
12535
12536 LLVM_DEBUG({
12537 dbgs() << "Case clusters: ";
12538 for (const CaseCluster &C : Clusters) {
12539 if (C.Kind == CC_JumpTable)
12540 dbgs() << "JT:";
12541 if (C.Kind == CC_BitTests)
12542 dbgs() << "BT:";
12543
12544 C.Low->getValue().print(dbgs(), true);
12545 if (C.Low != C.High) {
12546 dbgs() << '-';
12547 C.High->getValue().print(dbgs(), true);
12548 }
12549 dbgs() << ' ';
12550 }
12551 dbgs() << '\n';
12552 });
12553
12554 assert(!Clusters.empty());
12555 SwitchWorkList WorkList;
12556 CaseClusterIt First = Clusters.begin();
12557 CaseClusterIt Last = Clusters.end() - 1;
12558 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
12559 // Scale the branchprobability for DefaultMBB if the peel occurs and
12560 // DefaultMBB is not replaced.
12561 if (PeeledCaseProb != BranchProbability::getZero() &&
12562 DefaultMBB == FuncInfo.getMBB(SI.getDefaultDest()))
12563 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
12564 WorkList.push_back(
12565 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
12566
12567 while (!WorkList.empty()) {
12568 SwitchWorkListItem W = WorkList.pop_back_val();
12569 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12570
12571 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12572 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12573 // For optimized builds, lower large range as a balanced binary tree.
12574 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
12575 continue;
12576 }
12577
12578 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
12579 }
12580}
12581
12582void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12583 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12584 auto DL = getCurSDLoc();
12585 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12586 setValue(&I, DAG.getStepVector(DL, ResultVT));
12587}
12588
12589void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12590 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12591 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12592
12593 SDLoc DL = getCurSDLoc();
12594 SDValue V = getValue(I.getOperand(0));
12595 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12596
12597 if (VT.isScalableVector()) {
12598 setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
12599 return;
12600 }
12601
12602 // Use VECTOR_SHUFFLE for the fixed-length vector
12603 // to maintain existing behavior.
12604 SmallVector<int, 8> Mask;
12605 unsigned NumElts = VT.getVectorMinNumElements();
12606 for (unsigned i = 0; i != NumElts; ++i)
12607 Mask.push_back(NumElts - 1 - i);
12608
12609 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
12610}
12611
12612void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
12613 unsigned Factor) {
12614 auto DL = getCurSDLoc();
12615 SDValue InVec = getValue(I.getOperand(0));
12616
12617 SmallVector<EVT, 4> ValueVTs;
12618 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
12619 ValueVTs);
12620
12621 EVT OutVT = ValueVTs[0];
12622 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12623
12624 SmallVector<SDValue, 4> SubVecs(Factor);
12625 for (unsigned i = 0; i != Factor; ++i) {
12626 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
12627 SubVecs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12628 DAG.getVectorIdxConstant(OutNumElts * i, DL));
12629 }
12630
12631 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
12632 // from existing legalisation and combines.
12633 if (OutVT.isFixedLengthVector() && Factor == 2) {
12634 SDValue Even = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
12635 createStrideMask(0, 2, OutNumElts));
12636 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
12637 createStrideMask(1, 2, OutNumElts));
12638 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
12639 setValue(&I, Res);
12640 return;
12641 }
12642
12643 SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
12644 DAG.getVTList(ValueVTs), SubVecs);
12645 setValue(&I, Res);
12646}
12647
12648void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
12649 unsigned Factor) {
12650 auto DL = getCurSDLoc();
12651 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12652 EVT InVT = getValue(I.getOperand(0)).getValueType();
12653 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12654
12655 SmallVector<SDValue, 8> InVecs(Factor);
12656 for (unsigned i = 0; i < Factor; ++i) {
12657 InVecs[i] = getValue(I.getOperand(i));
12658 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
12659 "Expected VTs to be the same");
12660 }
12661
12662 // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
12663 // from existing legalisation and combines.
12664 if (OutVT.isFixedLengthVector() && Factor == 2) {
12665 unsigned NumElts = InVT.getVectorMinNumElements();
12666 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVecs);
12667 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
12668 createInterleaveMask(NumElts, 2)));
12669 return;
12670 }
12671
12672 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
12673 SDValue Res =
12674 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);
12675
12677 for (unsigned i = 0; i < Factor; ++i)
12678 Results[i] = Res.getValue(i);
12679
12680 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Results);
12681 setValue(&I, Res);
12682}
12683
12684void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12685 SmallVector<EVT, 4> ValueVTs;
12686 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
12687 ValueVTs);
12688 unsigned NumValues = ValueVTs.size();
12689 if (NumValues == 0) return;
12690
12691 SmallVector<SDValue, 4> Values(NumValues);
12692 SDValue Op = getValue(I.getOperand(0));
12693
12694 for (unsigned i = 0; i != NumValues; ++i)
12695 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
12696 SDValue(Op.getNode(), Op.getResNo() + i));
12697
12699 DAG.getVTList(ValueVTs), Values));
12700}
12701
12702void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12703 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12704 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12705
12706 SDLoc DL = getCurSDLoc();
12707 SDValue V1 = getValue(I.getOperand(0));
12708 SDValue V2 = getValue(I.getOperand(1));
12709 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
12710
12711 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
12712 if (VT.isScalableVector()) {
12713 setValue(
12714 &I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
12715 DAG.getSignedConstant(
12716 Imm, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
12717 return;
12718 }
12719
12720 unsigned NumElts = VT.getVectorNumElements();
12721
12722 uint64_t Idx = (NumElts + Imm) % NumElts;
12723
12724 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12725 SmallVector<int, 8> Mask;
12726 for (unsigned i = 0; i < NumElts; ++i)
12727 Mask.push_back(Idx + i);
12728 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
12729}
12730
12731// Consider the following MIR after SelectionDAG, which produces output in
12732// phyregs in the first case or virtregs in the second case.
12733//
12734// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12735// %5:gr32 = COPY $ebx
12736// %6:gr32 = COPY $edx
12737// %1:gr32 = COPY %6:gr32
12738// %0:gr32 = COPY %5:gr32
12739//
12740// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12741// %1:gr32 = COPY %6:gr32
12742// %0:gr32 = COPY %5:gr32
12743//
12744// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12745// Given %1, we'd like to return $edx in the first case and %6 in the second.
12746//
12747// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12748// to a single virtreg (such as %0). The remaining outputs monotonically
12749// increase in virtreg number from there. If a callbr has no outputs, then it
12750// should not have a corresponding callbr landingpad; in fact, the callbr
12751// landingpad would not even be able to refer to such a callbr.
12753 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
12754 // There is definitely at least one copy.
12755 assert(MI->getOpcode() == TargetOpcode::COPY &&
12756 "start of copy chain MUST be COPY");
12757 Reg = MI->getOperand(1).getReg();
12758
12759 // If the copied register in the first copy must be virtual.
12760 assert(Reg.isVirtual() && "expected COPY of virtual register");
12761 MI = MRI.def_begin(Reg)->getParent();
12762
12763 // There may be an optional second copy.
12764 if (MI->getOpcode() == TargetOpcode::COPY) {
12765 assert(Reg.isVirtual() && "expected COPY of virtual register");
12766 Reg = MI->getOperand(1).getReg();
12767 assert(Reg.isPhysical() && "expected COPY of physical register");
12768 } else {
12769 // The start of the chain must be an INLINEASM_BR.
12770 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12771 "end of copy chain MUST be INLINEASM_BR");
12772 }
12773
12774 return Reg;
12775}
12776
12777// We must do this walk rather than the simpler
12778// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
12779// otherwise we will end up with copies of virtregs only valid along direct
12780// edges.
12781void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
12782 SmallVector<EVT, 8> ResultVTs;
12783 SmallVector<SDValue, 8> ResultValues;
12784 const auto *CBR =
12785 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
12786
12787 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12788 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
12789 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
12790
12791 Register InitialDef = FuncInfo.ValueMap[CBR];
12792 SDValue Chain = DAG.getRoot();
12793
12794 // Re-parse the asm constraints string.
12795 TargetLowering::AsmOperandInfoVector TargetConstraints =
12796 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
12797 for (auto &T : TargetConstraints) {
12798 SDISelAsmOperandInfo OpInfo(T);
12799 if (OpInfo.Type != InlineAsm::isOutput)
12800 continue;
12801
12802 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
12803 // individual constraint.
12804 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
12805
12806 switch (OpInfo.ConstraintType) {
12809 // Fill in OpInfo.AssignedRegs.Regs.
12810 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
12811
12812 // getRegistersForValue may produce 1 to many registers based on whether
12813 // the OpInfo.ConstraintVT is legal on the target or not.
12814 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
12815 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
12816 if (OriginalDef.isPhysical())
12817 FuncInfo.MBB->addLiveIn(OriginalDef);
12818 // Update the assigned registers to use the original defs.
12819 Reg = OriginalDef;
12820 }
12821
12822 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
12823 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
12824 ResultValues.push_back(V);
12825 ResultVTs.push_back(OpInfo.ConstraintVT);
12826 break;
12827 }
12829 SDValue Flag;
12830 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
12831 OpInfo, DAG);
12832 ++InitialDef;
12833 ResultValues.push_back(V);
12834 ResultVTs.push_back(OpInfo.ConstraintVT);
12835 break;
12836 }
12837 default:
12838 break;
12839 }
12840 }
12842 DAG.getVTList(ResultVTs), ResultValues);
12843 setValue(&I, V);
12844}
unsigned const MachineRegisterInfo * MRI
return SDValue()
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
Definition CSEInfo.cpp:27
This file contains the declarations for the subclasses of Constant, which represent the different fla...
dxil translate DXIL Translate Metadata
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition FastISel.cpp:942
#define Check(C,...)
static Value * getCondition(Instruction *I)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
Machine Check Debug Module
static bool isUndef(const MachineInstr &MI)
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file contains the declarations for metadata subclasses.
Type::TypeID TypeID
#define T
#define T1
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
MachineInstr unsigned OpIdx
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
#define P(N)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static void failForInvalidBundles(const CallBase &I, StringRef Name, ArrayRef< uint32_t > AllowedBundles)
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
DenseMap< const Argument *, std::pair< const AllocaInst *, const StoreInst * > > ArgCopyElisionMapTy
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< Register, TypeSize > > &Regs, const SDValue &N)
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const unsigned char *MatcherTable, unsigned &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:114
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
Value * RHS
Value * LHS
static const fltSemantics & IEEEsingle()
Definition APFloat.h:296
Class for arbitrary precision integers.
Definition APInt.h:78
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:334
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:440
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition Function.cpp:339
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition Argument.h:50
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
iterator end() const
Definition ArrayRef.h:136
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
iterator begin() const
Definition ArrayRef.h:135
bool empty() const
empty - Check if the array is empty.
Definition ArrayRef.h:142
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
@ Add
*p = old + v
@ FAdd
*p = old + v
@ USubCond
Subtract only if no unsigned overflow.
@ FMinimum
*p = minimum(old, v) minimum matches the behavior of llvm.minimum.
@ Min
*p = old <signed v ? old : v
@ Sub
*p = old - v
@ And
*p = old & v
@ Xor
*p = old ^ v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ FMaximum
*p = maximum(old, v) maximum matches the behavior of llvm.maximum.
@ FSub
*p = old - v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
@ Nand
*p = ~(old & v)
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:361
LLVM Basic Block Representation.
Definition BasicBlock.h:62
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
InstListType::const_iterator const_iterator
Definition BasicBlock.h:171
LLVM_ABI bool isEntryBlock() const
Return true if this is the entry block of the containing function.
LLVM_ABI InstListType::const_iterator getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition BasicBlock.h:233
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
This class represents a no-op cast from one type to another.
The address of a basic block.
Definition Constants.h:899
Conditional or Unconditional Branch instruction.
Analysis providing branch probability information.
LLVM_ABI BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
LLVM_ABI bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static uint32_t getDenominator()
static BranchProbability getOne()
static BranchProbability getUnknown()
uint32_t getNumerator() const
LLVM_ABI uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
LLVM_ABI bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition InstrTypes.h:664
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:676
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition Constants.h:593
A constant value that is initialized with an expression using other constant values.
Definition Constants.h:1120
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:277
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:214
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:163
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:154
A signed pointer, in the ptrauth sense.
Definition Constants.h:1032
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition Constants.h:517
This is an important base class in LLVM.
Definition Constant.h:43
This is the common base class for constrained floating point intrinsics.
LLVM_ABI std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
LLVM_ABI unsigned getNonMetadataArgCount() const
DWARF expression.
LLVM_ABI bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static LLVM_ABI std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
LLVM_ABI uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static LLVM_ABI const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static LLVM_ABI DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
Base class for variables.
LLVM_ABI std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
bool isBigEndian() const
Definition DataLayout.h:208
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
LLVM_ABI Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
LLVM_ABI iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
A debug info location.
Definition DebugLoc.h:124
LLVM_ABI DILocation * getInlinedAt() const
Definition DebugLoc.cpp:69
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:167
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Definition DenseMap.h:74
bool empty() const
Definition DenseMap.h:109
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT, true > const_iterator
Definition DenseMap.h:75
iterator end()
Definition DenseMap.h:81
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:222
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition DenseMap.h:114
Diagnostic information for inline asm reporting.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:310
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:316
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:321
Lightweight error class with error context and mandatory checking.
Definition Error.h:159
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition FastISel.h:66
bool allowReassoc() const
Flag queries.
Definition FMF.h:64
An instruction for ordering other memory operations.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:803
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
MachineBasicBlock * getMBB(const BasicBlock *BB) const
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
MachineBasicBlock * MBB
MBB - The current block.
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition Function.h:807
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:209
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:244
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:703
bool hasParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const
check if an attributes is in the list of attributes.
Definition Function.cpp:739
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:270
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:352
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition Function.h:249
size_t arg_size() const
Definition Function.h:899
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:727
Garbage collection metadata for a single function.
Definition GCMetadata.h:80
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
This instruction inserts a struct field of array element value into an aggregate value.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
A helper class to return the specified delimiter string after the first invocation of operator String...
An instruction for reading from memory.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
LLVM_ABI MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
LLVM_ABI MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition MCSymbol.h:42
Metadata node.
Definition Metadata.h:1078
Machine Value Type.
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHContTarget(bool V=true)
Indicates if this is a target of Windows EH Continuation Guard.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
MachineInstrBundleIterator< MachineInstr > iterator
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
void setHasEHContTarget(bool V)
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI MCRegister getLiveInPhysReg(Register VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool contains(const KeyT &Key) const
Definition MapVector.h:141
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition MapVector.h:111
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:183
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Wrapper class representing virtual and physical registers.
Definition Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:74
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:78
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(Register VReg)
static SDDbgOperand fromConst(const Value *Const)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
DenseMap< const Constant *, Register > ConstantsOut
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const BranchInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, Register Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool canTailCall(const CallBase &CB) const
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void CopyValueToVirtualRegister(const Value *V, Register Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, const TargetLibraryInfo *li)
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineRegisterInfo * RegInfo
std::unique_ptr< SwiftErrorValueTracking > SwiftError
virtual void emitFunctionEntryCode()
std::unique_ptr< SelectionDAGBuilder > SDB
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, const CallInst *CI) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, const CallInst *CI) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI bool shouldOptForSize() const
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
LLVM_ABI void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
LLVM_ABI SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
LLVM_ABI SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void resize(size_type N)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitFunctionBasedCheckStackProtector() const
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
constexpr bool empty() const
empty - Check if the string is empty.
Definition StringRef.h:143
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:140
Multiway switch.
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Provides information about what library functions are available for the current target.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Returns the desired alignment for ByVal or InAlloca aggregate function arguments in the caller parame...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC, bool ZeroIsPoison, const ConstantRange *VScaleRange) const
Return the minimum number of bits required to hold the maximum possible number of trailing zero vecto...
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
MachineMemOperand::Flags getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const
Return true if the @llvm.experimental.vector.match intrinsic should be expanded for vector type ‘VT’ ...
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
std::vector< ArgListEntry > ArgListTy
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual bool supportPtrAuthBundles() const
Return true if the target supports ptrauth operand bundles.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual MVT getJumpTableRegTy(const DataLayout &DL) const
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const
Return a TargetTransformInfo for a given function.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
unsigned getID() const
Return the register class ID number.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
@ TCK_Latency
The latency of instruction.
LLVM_ABI bool hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:344
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
LLVM_ABI bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
Definition Type.cpp:181
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:273
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:267
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:281
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:128
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:294
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
bool isTokenTy() const
Return true if this is 'token'.
Definition Type.h:234
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:301
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:225
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:139
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_iterator op_begin()
Definition User.h:284
Value * getOperand(unsigned i) const
Definition User.h:232
unsigned getNumOperands() const
Definition User.h:254
op_iterator op_end()
Definition User.h:286
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
LLVM_ABI CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
static LLVM_ABI std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
LLVM_ABI MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
iterator_range< user_iterator > users()
Definition Value.h:426
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:701
bool use_empty() const
Definition Value.h:346
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1099
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
const ParentTy * getParent() const
Definition ilist_node.h:34
A raw_ostream that writes to an std::string.
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:41
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:256
@ CTLZ_ZERO_UNDEF
Definition ISDOpcodes.h:780
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:504
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition ISDOpcodes.h:45
@ LOOP_DEPENDENCE_RAW_MASK
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition ISDOpcodes.h:163
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:593
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:771
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:387
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:259
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:393
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:841
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:511
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:215
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition ISDOpcodes.h:167
@ GlobalAddress
Definition ISDOpcodes.h:88
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:868
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:577
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:410
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:744
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
Definition ISDOpcodes.h:521
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
Definition ISDOpcodes.h:508
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:249
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:400
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition ISDOpcodes.h:151
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:832
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition ISDOpcodes.h:117
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition ISDOpcodes.h:779
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:347
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:628
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:534
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:541
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:369
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:784
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:242
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:669
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:225
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:343
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition ISDOpcodes.h:958
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:762
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
Definition ISDOpcodes.h:78
@ PtrAuthGlobalAddress
A ptrauth constant.
Definition ISDOpcodes.h:100
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:607
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition ISDOpcodes.h:48
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition ISDOpcodes.h:134
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:569
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:838
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition ISDOpcodes.h:130
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition ISDOpcodes.h:379
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:351
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:724
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition ISDOpcodes.h:633
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:406
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:793
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition ISDOpcodes.h:145
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:110
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:493
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:914
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:736
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:200
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition ISDOpcodes.h:732
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition ISDOpcodes.h:420
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:236
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:558
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:53
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition ISDOpcodes.h:654
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:947
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition ISDOpcodes.h:696
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition ISDOpcodes.h:122
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:933
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition ISDOpcodes.h:157
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:844
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:62
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:527
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:360
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:617
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:208
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:549
@ LOOP_DEPENDENCE_WAR_MASK
Set rounding mode.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
IntrinsicID_match m_VScale()
Matches a call to llvm.vscale().
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
Offsets
Offsets in bytes from the start of the input buffer.
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
std::vector< CaseCluster > CaseClusterVector
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
CaseClusterVector::iterator CaseClusterIt
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition Dwarf.h:149
ExceptionBehavior
Exception behavior used for floating point operations.
Definition FPEnv.h:39
@ ebStrict
This corresponds to "fpexcept.strict".
Definition FPEnv.h:42
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition FPEnv.h:41
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition FPEnv.h:40
constexpr float log2ef
Definition MathExtras.h:51
constexpr double e
constexpr float ln2f
Definition MathExtras.h:49
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:344
@ Offset
Definition DWP.cpp:477
@ Length
Definition DWP.cpp:477
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition Analysis.cpp:241
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1725
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1655
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Done
Definition Threading.h:60
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:293
LLVM_ABI void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
static ConstantRange getRange(Value *Op, SCCPSolver &Solver, const SmallPtrSetImpl< Value * > &InsertedValues)
Helper for getting ranges from Solver.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2136
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:243
auto cast_or_null(const Y &Val)
Definition Casting.h:714
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
gep_type_iterator gep_type_end(const User *GEP)
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:154
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
Definition STLExtras.h:1150
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:202
void ComputeValueTypes(const DataLayout &DL, Type *Ty, SmallVectorImpl< Type * > &Types, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
Given an LLVM IR type, compute non-aggregate subtypes.
Definition Analysis.cpp:72
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1732
LLVM_ABI llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition STLExtras.h:852
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1622
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:167
generic_gep_type_iterator<> gep_type_iterator
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
auto succ_size(const MachineBasicBlock *BB)
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition STLExtras.h:300
LLVM_ABI ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition Analysis.cpp:207
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition Local.cpp:2274
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:71
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
TargetTransformInfo TTI
FunctionAddr VTableAddr uintptr_t uintptr_t Data
Definition InstrProf.h:189
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
LLVM_ABI llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:144
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition Analysis.cpp:543
DWARFExpression::Operation Op
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition Analysis.cpp:229
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition Analysis.cpp:119
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2120
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition Analysis.cpp:185
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1897
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:201
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2108
LLVM_ABI Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition Analysis.cpp:33
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:330
@ Default
The result values are uniform if and only if all operands are uniform.
Definition Uniformity.h:20
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:180
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:869
#define N
#define NC
Definition regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:77
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:395
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:74
uint64_t getScalarStoreSize() const
Definition ValueTypes.h:402
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:284
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:300
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:147
ElementCount getVectorElementCount() const
Definition ValueTypes.h:350
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:373
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:359
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:385
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:316
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:65
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
Definition ValueTypes.h:179
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:381
bool isFixedLengthVector() const
Definition ValueTypes.h:181
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:168
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:323
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:292
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:174
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:328
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:157
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:102
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:336
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:152
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition InlineAsm.h:128
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:248
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition Alignment.h:130
A lightweight accessor for an operand bundle meant to be passed around by value.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< std::pair< Register, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< Register, 4 > Regs
This list holds the registers assigned to the values.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
void setUnpredictable(bool b)
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
A MapVector that performs no allocations if smaller than a certain size.
Definition MapVector.h:257
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
Type * OrigRetTy
Original unlegalized return type.
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)