LLVM 19.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/Metadata.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Operator.h"
87#include "llvm/IR/Statepoint.h"
88#include "llvm/IR/Type.h"
89#include "llvm/IR/User.h"
90#include "llvm/IR/Value.h"
91#include "llvm/MC/MCContext.h"
96#include "llvm/Support/Debug.h"
105#include <cstddef>
106#include <iterator>
107#include <limits>
108#include <optional>
109#include <tuple>
110
111using namespace llvm;
112using namespace PatternMatch;
113using namespace SwitchCG;
114
115#define DEBUG_TYPE "isel"
116
117/// LimitFloatPrecision - Generate low-precision inline sequences for
118/// some float libcalls (6, 8 or 12 bits).
119static unsigned LimitFloatPrecision;
120
121static cl::opt<bool>
122 InsertAssertAlign("insert-assert-align", cl::init(true),
123 cl::desc("Insert the experimental `assertalign` node."),
125
127 LimitFPPrecision("limit-float-precision",
128 cl::desc("Generate low-precision inline sequences "
129 "for some float libcalls"),
131 cl::init(0));
132
134 "switch-peel-threshold", cl::Hidden, cl::init(66),
135 cl::desc("Set the case probability threshold for peeling the case from a "
136 "switch statement. A value greater than 100 will void this "
137 "optimization"));
138
139// Limit the width of DAG chains. This is important in general to prevent
140// DAG-based analysis from blowing up. For example, alias analysis and
141// load clustering may not complete in reasonable time. It is difficult to
142// recognize and avoid this situation within each individual analysis, and
143// future analyses are likely to have the same behavior. Limiting DAG width is
144// the safe approach and will be especially important with global DAGs.
145//
146// MaxParallelChains default is arbitrarily high to avoid affecting
147// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
148// sequence over this should have been converted to llvm.memcpy by the
149// frontend. It is easy to induce this behavior with .ll code such as:
150// %buffer = alloca [4096 x i8]
151// %data = load [4096 x i8]* %argPtr
152// store [4096 x i8] %data, [4096 x i8]* %buffer
153static const unsigned MaxParallelChains = 64;
154
156 const SDValue *Parts, unsigned NumParts,
157 MVT PartVT, EVT ValueVT, const Value *V,
158 SDValue InChain,
159 std::optional<CallingConv::ID> CC);
160
161/// getCopyFromParts - Create a value that contains the specified legal parts
162/// combined into the value they represent. If the parts combine to a type
163/// larger than ValueVT then AssertOp can be used to specify whether the extra
164/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
165/// (ISD::AssertSext).
166static SDValue
167getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
168 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
169 SDValue InChain,
170 std::optional<CallingConv::ID> CC = std::nullopt,
171 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
172 // Let the target assemble the parts if it wants to
173 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
174 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
175 PartVT, ValueVT, CC))
176 return Val;
177
178 if (ValueVT.isVector())
179 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
180 InChain, CC);
181
182 assert(NumParts > 0 && "No parts to assemble!");
183 SDValue Val = Parts[0];
184
185 if (NumParts > 1) {
186 // Assemble the value from multiple parts.
187 if (ValueVT.isInteger()) {
188 unsigned PartBits = PartVT.getSizeInBits();
189 unsigned ValueBits = ValueVT.getSizeInBits();
190
191 // Assemble the power of 2 part.
192 unsigned RoundParts = llvm::bit_floor(NumParts);
193 unsigned RoundBits = PartBits * RoundParts;
194 EVT RoundVT = RoundBits == ValueBits ?
195 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
196 SDValue Lo, Hi;
197
198 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
199
200 if (RoundParts > 2) {
201 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
202 InChain);
203 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
204 PartVT, HalfVT, V, InChain);
205 } else {
206 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
207 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
208 }
209
210 if (DAG.getDataLayout().isBigEndian())
211 std::swap(Lo, Hi);
212
213 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
214
215 if (RoundParts < NumParts) {
216 // Assemble the trailing non-power-of-2 part.
217 unsigned OddParts = NumParts - RoundParts;
218 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
219 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
220 OddVT, V, InChain, CC);
221
222 // Combine the round and odd parts.
223 Lo = Val;
224 if (DAG.getDataLayout().isBigEndian())
225 std::swap(Lo, Hi);
226 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
227 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
228 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
229 DAG.getConstant(Lo.getValueSizeInBits(), DL,
231 TotalVT, DAG.getDataLayout())));
232 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
233 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
234 }
235 } else if (PartVT.isFloatingPoint()) {
236 // FP split into multiple FP parts (for ppcf128)
237 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
238 "Unexpected split");
239 SDValue Lo, Hi;
240 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
241 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
242 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
243 std::swap(Lo, Hi);
244 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
245 } else {
246 // FP split into integer parts (soft fp)
247 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
248 !PartVT.isVector() && "Unexpected split");
249 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
250 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
251 InChain, CC);
252 }
253 }
254
255 // There is now one part, held in Val. Correct it to match ValueVT.
256 // PartEVT is the type of the register class that holds the value.
257 // ValueVT is the type of the inline asm operation.
258 EVT PartEVT = Val.getValueType();
259
260 if (PartEVT == ValueVT)
261 return Val;
262
263 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
264 ValueVT.bitsLT(PartEVT)) {
265 // For an FP value in an integer part, we need to truncate to the right
266 // width first.
267 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
268 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
269 }
270
271 // Handle types that have the same size.
272 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
273 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
274
275 // Handle types with different sizes.
276 if (PartEVT.isInteger() && ValueVT.isInteger()) {
277 if (ValueVT.bitsLT(PartEVT)) {
278 // For a truncate, see if we have any information to
279 // indicate whether the truncated bits will always be
280 // zero or sign-extension.
281 if (AssertOp)
282 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
283 DAG.getValueType(ValueVT));
284 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
285 }
286 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
287 }
288
289 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
290 // FP_ROUND's are always exact here.
291 if (ValueVT.bitsLT(Val.getValueType())) {
292
293 SDValue NoChange =
295
297 llvm::Attribute::StrictFP)) {
298 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
299 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
300 NoChange);
301 }
302
303 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
304 }
305
306 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
307 }
308
309 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
310 // then truncating.
311 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
312 ValueVT.bitsLT(PartEVT)) {
313 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
314 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
315 }
316
317 report_fatal_error("Unknown mismatch in getCopyFromParts!");
318}
319
321 const Twine &ErrMsg) {
322 const Instruction *I = dyn_cast_or_null<Instruction>(V);
323 if (!V)
324 return Ctx.emitError(ErrMsg);
325
326 const char *AsmError = ", possible invalid constraint for vector type";
327 if (const CallInst *CI = dyn_cast<CallInst>(I))
328 if (CI->isInlineAsm())
329 return Ctx.emitError(I, ErrMsg + AsmError);
330
331 return Ctx.emitError(I, ErrMsg);
332}
333
334/// getCopyFromPartsVector - Create a value that contains the specified legal
335/// parts combined into the value they represent. If the parts combine to a
336/// type larger than ValueVT then AssertOp can be used to specify whether the
337/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
338/// ValueVT (ISD::AssertSext).
340 const SDValue *Parts, unsigned NumParts,
341 MVT PartVT, EVT ValueVT, const Value *V,
342 SDValue InChain,
343 std::optional<CallingConv::ID> CallConv) {
344 assert(ValueVT.isVector() && "Not a vector value");
345 assert(NumParts > 0 && "No parts to assemble!");
346 const bool IsABIRegCopy = CallConv.has_value();
347
348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
349 SDValue Val = Parts[0];
350
351 // Handle a multi-element vector.
352 if (NumParts > 1) {
353 EVT IntermediateVT;
354 MVT RegisterVT;
355 unsigned NumIntermediates;
356 unsigned NumRegs;
357
358 if (IsABIRegCopy) {
360 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
361 NumIntermediates, RegisterVT);
362 } else {
363 NumRegs =
364 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
365 NumIntermediates, RegisterVT);
366 }
367
368 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
369 NumParts = NumRegs; // Silence a compiler warning.
370 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
371 assert(RegisterVT.getSizeInBits() ==
372 Parts[0].getSimpleValueType().getSizeInBits() &&
373 "Part type sizes don't match!");
374
375 // Assemble the parts into intermediate operands.
376 SmallVector<SDValue, 8> Ops(NumIntermediates);
377 if (NumIntermediates == NumParts) {
378 // If the register was not expanded, truncate or copy the value,
379 // as appropriate.
380 for (unsigned i = 0; i != NumParts; ++i)
381 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
382 V, InChain, CallConv);
383 } else if (NumParts > 0) {
384 // If the intermediate type was expanded, build the intermediate
385 // operands from the parts.
386 assert(NumParts % NumIntermediates == 0 &&
387 "Must expand into a divisible number of parts!");
388 unsigned Factor = NumParts / NumIntermediates;
389 for (unsigned i = 0; i != NumIntermediates; ++i)
390 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
391 IntermediateVT, V, InChain, CallConv);
392 }
393
394 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
395 // intermediate operands.
396 EVT BuiltVectorTy =
397 IntermediateVT.isVector()
399 *DAG.getContext(), IntermediateVT.getScalarType(),
400 IntermediateVT.getVectorElementCount() * NumParts)
402 IntermediateVT.getScalarType(),
403 NumIntermediates);
404 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
406 DL, BuiltVectorTy, Ops);
407 }
408
409 // There is now one part, held in Val. Correct it to match ValueVT.
410 EVT PartEVT = Val.getValueType();
411
412 if (PartEVT == ValueVT)
413 return Val;
414
415 if (PartEVT.isVector()) {
416 // Vector/Vector bitcast.
417 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
418 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
419
420 // If the parts vector has more elements than the value vector, then we
421 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
422 // Extract the elements we want.
423 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
426 (PartEVT.getVectorElementCount().isScalable() ==
427 ValueVT.getVectorElementCount().isScalable()) &&
428 "Cannot narrow, it would be a lossy transformation");
429 PartEVT =
431 ValueVT.getVectorElementCount());
432 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
433 DAG.getVectorIdxConstant(0, DL));
434 if (PartEVT == ValueVT)
435 return Val;
436 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
437 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
438
439 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
440 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
441 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
442 }
443
444 // Promoted vector extract
445 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
446 }
447
448 // Trivial bitcast if the types are the same size and the destination
449 // vector type is legal.
450 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
451 TLI.isTypeLegal(ValueVT))
452 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
453
454 if (ValueVT.getVectorNumElements() != 1) {
455 // Certain ABIs require that vectors are passed as integers. For vectors
456 // are the same size, this is an obvious bitcast.
457 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
458 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
459 } else if (ValueVT.bitsLT(PartEVT)) {
460 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
461 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
462 // Drop the extra bits.
463 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
464 return DAG.getBitcast(ValueVT, Val);
465 }
466
468 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
469 return DAG.getUNDEF(ValueVT);
470 }
471
472 // Handle cases such as i8 -> <1 x i1>
473 EVT ValueSVT = ValueVT.getVectorElementType();
474 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
475 unsigned ValueSize = ValueSVT.getSizeInBits();
476 if (ValueSize == PartEVT.getSizeInBits()) {
477 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
478 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
479 // It's possible a scalar floating point type gets softened to integer and
480 // then promoted to a larger integer. If PartEVT is the larger integer
481 // we need to truncate it and then bitcast to the FP type.
482 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
483 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
484 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
485 Val = DAG.getBitcast(ValueSVT, Val);
486 } else {
487 Val = ValueVT.isFloatingPoint()
488 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
489 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
490 }
491 }
492
493 return DAG.getBuildVector(ValueVT, DL, Val);
494}
495
496static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
497 SDValue Val, SDValue *Parts, unsigned NumParts,
498 MVT PartVT, const Value *V,
499 std::optional<CallingConv::ID> CallConv);
500
501/// getCopyToParts - Create a series of nodes that contain the specified value
502/// split into legal parts. If the parts contain more bits than Val, then, for
503/// integers, ExtendKind can be used to specify how to generate the extra bits.
504static void
506 unsigned NumParts, MVT PartVT, const Value *V,
507 std::optional<CallingConv::ID> CallConv = std::nullopt,
508 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
509 // Let the target split the parts if it wants to
510 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
511 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
512 CallConv))
513 return;
514 EVT ValueVT = Val.getValueType();
515
516 // Handle the vector case separately.
517 if (ValueVT.isVector())
518 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
519 CallConv);
520
521 unsigned OrigNumParts = NumParts;
523 "Copying to an illegal type!");
524
525 if (NumParts == 0)
526 return;
527
528 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
529 EVT PartEVT = PartVT;
530 if (PartEVT == ValueVT) {
531 assert(NumParts == 1 && "No-op copy with multiple parts!");
532 Parts[0] = Val;
533 return;
534 }
535
536 unsigned PartBits = PartVT.getSizeInBits();
537 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
538 // If the parts cover more bits than the value has, promote the value.
539 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
540 assert(NumParts == 1 && "Do not know what to promote to!");
541 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
542 } else {
543 if (ValueVT.isFloatingPoint()) {
544 // FP values need to be bitcast, then extended if they are being put
545 // into a larger container.
546 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
547 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
548 }
549 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
550 ValueVT.isInteger() &&
551 "Unknown mismatch!");
552 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
553 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
554 if (PartVT == MVT::x86mmx)
555 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
556 }
557 } else if (PartBits == ValueVT.getSizeInBits()) {
558 // Different types of the same size.
559 assert(NumParts == 1 && PartEVT != ValueVT);
560 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
561 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
562 // If the parts cover less bits than value has, truncate the value.
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
567 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
570 }
571
572 // The value may have changed - recompute ValueVT.
573 ValueVT = Val.getValueType();
574 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
575 "Failed to tile the value with PartVT!");
576
577 if (NumParts == 1) {
578 if (PartEVT != ValueVT) {
580 "scalar-to-vector conversion failed");
581 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
582 }
583
584 Parts[0] = Val;
585 return;
586 }
587
588 // Expand the value into multiple parts.
589 if (NumParts & (NumParts - 1)) {
590 // The number of parts is not a power of 2. Split off and copy the tail.
591 assert(PartVT.isInteger() && ValueVT.isInteger() &&
592 "Do not know what to expand to!");
593 unsigned RoundParts = llvm::bit_floor(NumParts);
594 unsigned RoundBits = RoundParts * PartBits;
595 unsigned OddParts = NumParts - RoundParts;
596 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
597 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
598
599 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
600 CallConv);
601
602 if (DAG.getDataLayout().isBigEndian())
603 // The odd parts were reversed by getCopyToParts - unreverse them.
604 std::reverse(Parts + RoundParts, Parts + NumParts);
605
606 NumParts = RoundParts;
607 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
608 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
609 }
610
611 // The number of parts is a power of 2. Repeatedly bisect the value using
612 // EXTRACT_ELEMENT.
613 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
615 ValueVT.getSizeInBits()),
616 Val);
617
618 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
619 for (unsigned i = 0; i < NumParts; i += StepSize) {
620 unsigned ThisBits = StepSize * PartBits / 2;
621 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
622 SDValue &Part0 = Parts[i];
623 SDValue &Part1 = Parts[i+StepSize/2];
624
625 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
626 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
627 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
628 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
629
630 if (ThisBits == PartBits && ThisVT != PartVT) {
631 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
632 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
633 }
634 }
635 }
636
637 if (DAG.getDataLayout().isBigEndian())
638 std::reverse(Parts, Parts + OrigNumParts);
639}
640
642 const SDLoc &DL, EVT PartVT) {
643 if (!PartVT.isVector())
644 return SDValue();
645
646 EVT ValueVT = Val.getValueType();
647 EVT PartEVT = PartVT.getVectorElementType();
648 EVT ValueEVT = ValueVT.getVectorElementType();
649 ElementCount PartNumElts = PartVT.getVectorElementCount();
650 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
651
652 // We only support widening vectors with equivalent element types and
653 // fixed/scalable properties. If a target needs to widen a fixed-length type
654 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
655 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
656 PartNumElts.isScalable() != ValueNumElts.isScalable())
657 return SDValue();
658
659 // Have a try for bf16 because some targets share its ABI with fp16.
660 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
662 "Cannot widen to illegal type");
663 Val = DAG.getNode(ISD::BITCAST, DL,
664 ValueVT.changeVectorElementType(MVT::f16), Val);
665 } else if (PartEVT != ValueEVT) {
666 return SDValue();
667 }
668
669 // Widening a scalable vector to another scalable vector is done by inserting
670 // the vector into a larger undef one.
671 if (PartNumElts.isScalable())
672 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
673 Val, DAG.getVectorIdxConstant(0, DL));
674
675 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
676 // undef elements.
678 DAG.ExtractVectorElements(Val, Ops);
679 SDValue EltUndef = DAG.getUNDEF(PartEVT);
680 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
681
682 // FIXME: Use CONCAT for 2x -> 4x.
683 return DAG.getBuildVector(PartVT, DL, Ops);
684}
685
686/// getCopyToPartsVector - Create a series of nodes that contain the specified
687/// value split into legal parts.
688static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
689 SDValue Val, SDValue *Parts, unsigned NumParts,
690 MVT PartVT, const Value *V,
691 std::optional<CallingConv::ID> CallConv) {
692 EVT ValueVT = Val.getValueType();
693 assert(ValueVT.isVector() && "Not a vector");
694 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
695 const bool IsABIRegCopy = CallConv.has_value();
696
697 if (NumParts == 1) {
698 EVT PartEVT = PartVT;
699 if (PartEVT == ValueVT) {
700 // Nothing to do.
701 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
702 // Bitconvert vector->vector case.
703 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
704 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
705 Val = Widened;
706 } else if (PartVT.isVector() &&
708 ValueVT.getVectorElementType()) &&
709 PartEVT.getVectorElementCount() ==
710 ValueVT.getVectorElementCount()) {
711
712 // Promoted vector extract
713 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
714 } else if (PartEVT.isVector() &&
715 PartEVT.getVectorElementType() !=
716 ValueVT.getVectorElementType() &&
717 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
718 TargetLowering::TypeWidenVector) {
719 // Combination of widening and promotion.
720 EVT WidenVT =
722 PartVT.getVectorElementCount());
723 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
724 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
725 } else {
726 // Don't extract an integer from a float vector. This can happen if the
727 // FP type gets softened to integer and then promoted. The promotion
728 // prevents it from being picked up by the earlier bitcast case.
729 if (ValueVT.getVectorElementCount().isScalar() &&
730 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
731 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
732 DAG.getVectorIdxConstant(0, DL));
733 } else {
734 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
735 assert(PartVT.getFixedSizeInBits() > ValueSize &&
736 "lossy conversion of vector to scalar type");
737 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
738 Val = DAG.getBitcast(IntermediateType, Val);
739 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
740 }
741 }
742
743 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
744 Parts[0] = Val;
745 return;
746 }
747
748 // Handle a multi-element vector.
749 EVT IntermediateVT;
750 MVT RegisterVT;
751 unsigned NumIntermediates;
752 unsigned NumRegs;
753 if (IsABIRegCopy) {
755 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
756 RegisterVT);
757 } else {
758 NumRegs =
759 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
760 NumIntermediates, RegisterVT);
761 }
762
763 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
764 NumParts = NumRegs; // Silence a compiler warning.
765 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
766
767 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
768 "Mixing scalable and fixed vectors when copying in parts");
769
770 std::optional<ElementCount> DestEltCnt;
771
772 if (IntermediateVT.isVector())
773 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
774 else
775 DestEltCnt = ElementCount::getFixed(NumIntermediates);
776
777 EVT BuiltVectorTy = EVT::getVectorVT(
778 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
779
780 if (ValueVT == BuiltVectorTy) {
781 // Nothing to do.
782 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
783 // Bitconvert vector->vector case.
784 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
785 } else {
786 if (BuiltVectorTy.getVectorElementType().bitsGT(
787 ValueVT.getVectorElementType())) {
788 // Integer promotion.
789 ValueVT = EVT::getVectorVT(*DAG.getContext(),
790 BuiltVectorTy.getVectorElementType(),
791 ValueVT.getVectorElementCount());
792 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
793 }
794
795 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
796 Val = Widened;
797 }
798 }
799
800 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
801
802 // Split the vector into intermediate operands.
803 SmallVector<SDValue, 8> Ops(NumIntermediates);
804 for (unsigned i = 0; i != NumIntermediates; ++i) {
805 if (IntermediateVT.isVector()) {
806 // This does something sensible for scalable vectors - see the
807 // definition of EXTRACT_SUBVECTOR for further details.
808 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
809 Ops[i] =
810 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
811 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
812 } else {
813 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
814 DAG.getVectorIdxConstant(i, DL));
815 }
816 }
817
818 // Split the intermediate operands into legal parts.
819 if (NumParts == NumIntermediates) {
820 // If the register was not expanded, promote or copy the value,
821 // as appropriate.
822 for (unsigned i = 0; i != NumParts; ++i)
823 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
824 } else if (NumParts > 0) {
825 // If the intermediate type was expanded, split each the value into
826 // legal parts.
827 assert(NumIntermediates != 0 && "division by zero");
828 assert(NumParts % NumIntermediates == 0 &&
829 "Must expand into a divisible number of parts!");
830 unsigned Factor = NumParts / NumIntermediates;
831 for (unsigned i = 0; i != NumIntermediates; ++i)
832 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
833 CallConv);
834 }
835}
836
838 EVT valuevt, std::optional<CallingConv::ID> CC)
839 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
840 RegCount(1, regs.size()), CallConv(CC) {}
841
843 const DataLayout &DL, unsigned Reg, Type *Ty,
844 std::optional<CallingConv::ID> CC) {
845 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
846
847 CallConv = CC;
848
849 for (EVT ValueVT : ValueVTs) {
850 unsigned NumRegs =
853 : TLI.getNumRegisters(Context, ValueVT);
854 MVT RegisterVT =
857 : TLI.getRegisterType(Context, ValueVT);
858 for (unsigned i = 0; i != NumRegs; ++i)
859 Regs.push_back(Reg + i);
860 RegVTs.push_back(RegisterVT);
861 RegCount.push_back(NumRegs);
862 Reg += NumRegs;
863 }
864}
865
867 FunctionLoweringInfo &FuncInfo,
868 const SDLoc &dl, SDValue &Chain,
869 SDValue *Glue, const Value *V) const {
870 // A Value with type {} or [0 x %t] needs no registers.
871 if (ValueVTs.empty())
872 return SDValue();
873
874 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
875
876 // Assemble the legal parts into the final values.
877 SmallVector<SDValue, 4> Values(ValueVTs.size());
879 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
880 // Copy the legal parts from the registers.
881 EVT ValueVT = ValueVTs[Value];
882 unsigned NumRegs = RegCount[Value];
883 MVT RegisterVT = isABIMangled()
885 *DAG.getContext(), *CallConv, RegVTs[Value])
886 : RegVTs[Value];
887
888 Parts.resize(NumRegs);
889 for (unsigned i = 0; i != NumRegs; ++i) {
890 SDValue P;
891 if (!Glue) {
892 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
893 } else {
894 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
895 *Glue = P.getValue(2);
896 }
897
898 Chain = P.getValue(1);
899 Parts[i] = P;
900
901 // If the source register was virtual and if we know something about it,
902 // add an assert node.
903 if (!Register::isVirtualRegister(Regs[Part + i]) ||
904 !RegisterVT.isInteger())
905 continue;
906
908 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
909 if (!LOI)
910 continue;
911
912 unsigned RegSize = RegisterVT.getScalarSizeInBits();
913 unsigned NumSignBits = LOI->NumSignBits;
914 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
915
916 if (NumZeroBits == RegSize) {
917 // The current value is a zero.
918 // Explicitly express that as it would be easier for
919 // optimizations to kick in.
920 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
921 continue;
922 }
923
924 // FIXME: We capture more information than the dag can represent. For
925 // now, just use the tightest assertzext/assertsext possible.
926 bool isSExt;
927 EVT FromVT(MVT::Other);
928 if (NumZeroBits) {
929 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
930 isSExt = false;
931 } else if (NumSignBits > 1) {
932 FromVT =
933 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
934 isSExt = true;
935 } else {
936 continue;
937 }
938 // Add an assertion node.
939 assert(FromVT != MVT::Other);
940 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
941 RegisterVT, P, DAG.getValueType(FromVT));
942 }
943
944 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
945 RegisterVT, ValueVT, V, Chain, CallConv);
946 Part += NumRegs;
947 Parts.clear();
948 }
949
950 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
951}
952
954 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
955 const Value *V,
956 ISD::NodeType PreferredExtendType) const {
957 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
958 ISD::NodeType ExtendKind = PreferredExtendType;
959
960 // Get the list of the values's legal parts.
961 unsigned NumRegs = Regs.size();
962 SmallVector<SDValue, 8> Parts(NumRegs);
963 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
964 unsigned NumParts = RegCount[Value];
965
966 MVT RegisterVT = isABIMangled()
968 *DAG.getContext(), *CallConv, RegVTs[Value])
969 : RegVTs[Value];
970
971 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
972 ExtendKind = ISD::ZERO_EXTEND;
973
974 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
975 NumParts, RegisterVT, V, CallConv, ExtendKind);
976 Part += NumParts;
977 }
978
979 // Copy the parts into the registers.
980 SmallVector<SDValue, 8> Chains(NumRegs);
981 for (unsigned i = 0; i != NumRegs; ++i) {
982 SDValue Part;
983 if (!Glue) {
984 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
985 } else {
986 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
987 *Glue = Part.getValue(1);
988 }
989
990 Chains[i] = Part.getValue(0);
991 }
992
993 if (NumRegs == 1 || Glue)
994 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
995 // flagged to it. That is the CopyToReg nodes and the user are considered
996 // a single scheduling unit. If we create a TokenFactor and return it as
997 // chain, then the TokenFactor is both a predecessor (operand) of the
998 // user as well as a successor (the TF operands are flagged to the user).
999 // c1, f1 = CopyToReg
1000 // c2, f2 = CopyToReg
1001 // c3 = TokenFactor c1, c2
1002 // ...
1003 // = op c3, ..., f2
1004 Chain = Chains[NumRegs-1];
1005 else
1006 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1007}
1008
1010 unsigned MatchingIdx, const SDLoc &dl,
1011 SelectionDAG &DAG,
1012 std::vector<SDValue> &Ops) const {
1013 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1014
1015 InlineAsm::Flag Flag(Code, Regs.size());
1016 if (HasMatching)
1017 Flag.setMatchingOp(MatchingIdx);
1018 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
1019 // Put the register class of the virtual registers in the flag word. That
1020 // way, later passes can recompute register class constraints for inline
1021 // assembly as well as normal instructions.
1022 // Don't do this for tied operands that can use the regclass information
1023 // from the def.
1025 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1026 Flag.setRegClass(RC->getID());
1027 }
1028
1029 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1030 Ops.push_back(Res);
1031
1032 if (Code == InlineAsm::Kind::Clobber) {
1033 // Clobbers should always have a 1:1 mapping with registers, and may
1034 // reference registers that have illegal (e.g. vector) types. Hence, we
1035 // shouldn't try to apply any sort of splitting logic to them.
1036 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1037 "No 1:1 mapping from clobbers to regs?");
1039 (void)SP;
1040 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1041 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1042 assert(
1043 (Regs[I] != SP ||
1045 "If we clobbered the stack pointer, MFI should know about it.");
1046 }
1047 return;
1048 }
1049
1050 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1051 MVT RegisterVT = RegVTs[Value];
1052 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1053 RegisterVT);
1054 for (unsigned i = 0; i != NumRegs; ++i) {
1055 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1056 unsigned TheReg = Regs[Reg++];
1057 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1058 }
1059 }
1060}
1061
1065 unsigned I = 0;
1066 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1067 unsigned RegCount = std::get<0>(CountAndVT);
1068 MVT RegisterVT = std::get<1>(CountAndVT);
1069 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1070 for (unsigned E = I + RegCount; I != E; ++I)
1071 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1072 }
1073 return OutVec;
1074}
1075
1077 AssumptionCache *ac,
1078 const TargetLibraryInfo *li) {
1079 AA = aa;
1080 AC = ac;
1081 GFI = gfi;
1082 LibInfo = li;
1084 LPadToCallSiteMap.clear();
1086 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1088}
1089
1091 NodeMap.clear();
1092 UnusedArgNodeMap.clear();
1093 PendingLoads.clear();
1094 PendingExports.clear();
1095 PendingConstrainedFP.clear();
1096 PendingConstrainedFPStrict.clear();
1097 CurInst = nullptr;
1098 HasTailCall = false;
1099 SDNodeOrder = LowestSDNodeOrder;
1101}
1102
1104 DanglingDebugInfoMap.clear();
1105}
1106
1107// Update DAG root to include dependencies on Pending chains.
1108SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1109 SDValue Root = DAG.getRoot();
1110
1111 if (Pending.empty())
1112 return Root;
1113
1114 // Add current root to PendingChains, unless we already indirectly
1115 // depend on it.
1116 if (Root.getOpcode() != ISD::EntryToken) {
1117 unsigned i = 0, e = Pending.size();
1118 for (; i != e; ++i) {
1119 assert(Pending[i].getNode()->getNumOperands() > 1);
1120 if (Pending[i].getNode()->getOperand(0) == Root)
1121 break; // Don't add the root if we already indirectly depend on it.
1122 }
1123
1124 if (i == e)
1125 Pending.push_back(Root);
1126 }
1127
1128 if (Pending.size() == 1)
1129 Root = Pending[0];
1130 else
1131 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1132
1133 DAG.setRoot(Root);
1134 Pending.clear();
1135 return Root;
1136}
1137
1139 return updateRoot(PendingLoads);
1140}
1141
1143 // Chain up all pending constrained intrinsics together with all
1144 // pending loads, by simply appending them to PendingLoads and
1145 // then calling getMemoryRoot().
1146 PendingLoads.reserve(PendingLoads.size() +
1147 PendingConstrainedFP.size() +
1148 PendingConstrainedFPStrict.size());
1149 PendingLoads.append(PendingConstrainedFP.begin(),
1150 PendingConstrainedFP.end());
1151 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1152 PendingConstrainedFPStrict.end());
1153 PendingConstrainedFP.clear();
1154 PendingConstrainedFPStrict.clear();
1155 return getMemoryRoot();
1156}
1157
1159 // We need to emit pending fpexcept.strict constrained intrinsics,
1160 // so append them to the PendingExports list.
1161 PendingExports.append(PendingConstrainedFPStrict.begin(),
1162 PendingConstrainedFPStrict.end());
1163 PendingConstrainedFPStrict.clear();
1164 return updateRoot(PendingExports);
1165}
1166
1168 DILocalVariable *Variable,
1170 DebugLoc DL) {
1171 assert(Variable && "Missing variable");
1172
1173 // Check if address has undef value.
1174 if (!Address || isa<UndefValue>(Address) ||
1175 (Address->use_empty() && !isa<Argument>(Address))) {
1176 LLVM_DEBUG(
1177 dbgs()
1178 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1179 return;
1180 }
1181
1182 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1183
1184 SDValue &N = NodeMap[Address];
1185 if (!N.getNode() && isa<Argument>(Address))
1186 // Check unused arguments map.
1187 N = UnusedArgNodeMap[Address];
1188 SDDbgValue *SDV;
1189 if (N.getNode()) {
1190 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1191 Address = BCI->getOperand(0);
1192 // Parameters are handled specially.
1193 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1194 if (IsParameter && FINode) {
1195 // Byval parameter. We have a frame index at this point.
1196 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1197 /*IsIndirect*/ true, DL, SDNodeOrder);
1198 } else if (isa<Argument>(Address)) {
1199 // Address is an argument, so try to emit its dbg value using
1200 // virtual register info from the FuncInfo.ValueMap.
1201 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1202 FuncArgumentDbgValueKind::Declare, N);
1203 return;
1204 } else {
1205 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1206 true, DL, SDNodeOrder);
1207 }
1208 DAG.AddDbgValue(SDV, IsParameter);
1209 } else {
1210 // If Address is an argument then try to emit its dbg value using
1211 // virtual register info from the FuncInfo.ValueMap.
1212 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1213 FuncArgumentDbgValueKind::Declare, N)) {
1214 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1215 << " (could not emit func-arg dbg_value)\n");
1216 }
1217 }
1218 return;
1219}
1220
1222 // Add SDDbgValue nodes for any var locs here. Do so before updating
1223 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1224 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1225 // Add SDDbgValue nodes for any var locs here. Do so before updating
1226 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1227 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1228 It != End; ++It) {
1229 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1230 dropDanglingDebugInfo(Var, It->Expr);
1231 if (It->Values.isKillLocation(It->Expr)) {
1232 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1233 continue;
1234 }
1235 SmallVector<Value *> Values(It->Values.location_ops());
1236 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1237 It->Values.hasArgList())) {
1239 for (Value *V : It->Values.location_ops())
1240 Vals.push_back(V);
1242 FnVarLocs->getDILocalVariable(It->VariableID),
1243 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1244 }
1245 }
1246 }
1247
1248 // We must skip DPValues if they've already been processed above as we
1249 // have just emitted the debug values resulting from assignment tracking
1250 // analysis, making any existing DPValues redundant (and probably less
1251 // correct). We still need to process DPLabels. This does sink DPLabels
1252 // to the bottom of the group of debug records. That sholdn't be important
1253 // as it does so deterministcally and ordering between DPLabels and DPValues
1254 // is immaterial (other than for MIR/IR printing).
1255 bool SkipDPValues = DAG.getFunctionVarLocs();
1256 // Is there is any debug-info attached to this instruction, in the form of
1257 // DbgRecord non-instruction debug-info records.
1258 for (DbgRecord &DR : I.getDbgRecordRange()) {
1259 if (DPLabel *DPL = dyn_cast<DPLabel>(&DR)) {
1260 assert(DPL->getLabel() && "Missing label");
1261 SDDbgLabel *SDV =
1262 DAG.getDbgLabel(DPL->getLabel(), DPL->getDebugLoc(), SDNodeOrder);
1263 DAG.AddDbgLabel(SDV);
1264 continue;
1265 }
1266
1267 if (SkipDPValues)
1268 continue;
1269 DPValue &DPV = cast<DPValue>(DR);
1270 DILocalVariable *Variable = DPV.getVariable();
1273
1275 if (FuncInfo.PreprocessedDPVDeclares.contains(&DPV))
1276 continue;
1277 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DPV
1278 << "\n");
1280 DPV.getDebugLoc());
1281 continue;
1282 }
1283
1284 // A DPValue with no locations is a kill location.
1286 if (Values.empty()) {
1288 SDNodeOrder);
1289 continue;
1290 }
1291
1292 // A DPValue with an undef or absent location is also a kill location.
1293 if (llvm::any_of(Values,
1294 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1296 SDNodeOrder);
1297 continue;
1298 }
1299
1300 bool IsVariadic = DPV.hasArgList();
1301 if (!handleDebugValue(Values, Variable, Expression, DPV.getDebugLoc(),
1302 SDNodeOrder, IsVariadic)) {
1303 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1304 DPV.getDebugLoc(), SDNodeOrder);
1305 }
1306 }
1307}
1308
1310 visitDbgInfo(I);
1311
1312 // Set up outgoing PHI node register values before emitting the terminator.
1313 if (I.isTerminator()) {
1314 HandlePHINodesInSuccessorBlocks(I.getParent());
1315 }
1316
1317 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1318 if (!isa<DbgInfoIntrinsic>(I))
1319 ++SDNodeOrder;
1320
1321 CurInst = &I;
1322
1323 // Set inserted listener only if required.
1324 bool NodeInserted = false;
1325 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1326 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1327 if (PCSectionsMD) {
1328 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1329 DAG, [&](SDNode *) { NodeInserted = true; });
1330 }
1331
1332 visit(I.getOpcode(), I);
1333
1334 if (!I.isTerminator() && !HasTailCall &&
1335 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1337
1338 // Handle metadata.
1339 if (PCSectionsMD) {
1340 auto It = NodeMap.find(&I);
1341 if (It != NodeMap.end()) {
1342 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1343 } else if (NodeInserted) {
1344 // This should not happen; if it does, don't let it go unnoticed so we can
1345 // fix it. Relevant visit*() function is probably missing a setValue().
1346 errs() << "warning: loosing !pcsections metadata ["
1347 << I.getModule()->getName() << "]\n";
1348 LLVM_DEBUG(I.dump());
1349 assert(false);
1350 }
1351 }
1352
1353 CurInst = nullptr;
1354}
1355
1356void SelectionDAGBuilder::visitPHI(const PHINode &) {
1357 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1358}
1359
1360void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1361 // Note: this doesn't use InstVisitor, because it has to work with
1362 // ConstantExpr's in addition to instructions.
1363 switch (Opcode) {
1364 default: llvm_unreachable("Unknown instruction type encountered!");
1365 // Build the switch statement using the Instruction.def file.
1366#define HANDLE_INST(NUM, OPCODE, CLASS) \
1367 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1368#include "llvm/IR/Instruction.def"
1369 }
1370}
1371
1373 DILocalVariable *Variable,
1374 DebugLoc DL, unsigned Order,
1377 // For variadic dbg_values we will now insert an undef.
1378 // FIXME: We can potentially recover these!
1380 for (const Value *V : Values) {
1381 auto *Undef = UndefValue::get(V->getType());
1383 }
1384 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1385 /*IsIndirect=*/false, DL, Order,
1386 /*IsVariadic=*/true);
1387 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1388 return true;
1389}
1390
1392 DILocalVariable *Var,
1393 DIExpression *Expr,
1394 bool IsVariadic, DebugLoc DL,
1395 unsigned Order) {
1396 if (IsVariadic) {
1397 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1398 return;
1399 }
1400 // TODO: Dangling debug info will eventually either be resolved or produce
1401 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1402 // between the original dbg.value location and its resolved DBG_VALUE,
1403 // which we should ideally fill with an extra Undef DBG_VALUE.
1404 assert(Values.size() == 1);
1405 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1406}
1407
1409 const DIExpression *Expr) {
1410 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1411 DIVariable *DanglingVariable = DDI.getVariable();
1412 DIExpression *DanglingExpr = DDI.getExpression();
1413 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1414 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1415 << printDDI(nullptr, DDI) << "\n");
1416 return true;
1417 }
1418 return false;
1419 };
1420
1421 for (auto &DDIMI : DanglingDebugInfoMap) {
1422 DanglingDebugInfoVector &DDIV = DDIMI.second;
1423
1424 // If debug info is to be dropped, run it through final checks to see
1425 // whether it can be salvaged.
1426 for (auto &DDI : DDIV)
1427 if (isMatchingDbgValue(DDI))
1428 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1429
1430 erase_if(DDIV, isMatchingDbgValue);
1431 }
1432}
1433
1434// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1435// generate the debug data structures now that we've seen its definition.
1437 SDValue Val) {
1438 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1439 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1440 return;
1441
1442 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1443 for (auto &DDI : DDIV) {
1444 DebugLoc DL = DDI.getDebugLoc();
1445 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1446 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1447 DILocalVariable *Variable = DDI.getVariable();
1448 DIExpression *Expr = DDI.getExpression();
1450 "Expected inlined-at fields to agree");
1451 SDDbgValue *SDV;
1452 if (Val.getNode()) {
1453 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1454 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1455 // we couldn't resolve it directly when examining the DbgValue intrinsic
1456 // in the first place we should not be more successful here). Unless we
1457 // have some test case that prove this to be correct we should avoid
1458 // calling EmitFuncArgumentDbgValue here.
1459 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1460 FuncArgumentDbgValueKind::Value, Val)) {
1461 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1462 << printDDI(V, DDI) << "\n");
1463 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1464 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1465 // inserted after the definition of Val when emitting the instructions
1466 // after ISel. An alternative could be to teach
1467 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1468 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1469 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1470 << ValSDNodeOrder << "\n");
1471 SDV = getDbgValue(Val, Variable, Expr, DL,
1472 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1473 DAG.AddDbgValue(SDV, false);
1474 } else
1475 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1476 << printDDI(V, DDI)
1477 << " in EmitFuncArgumentDbgValue\n");
1478 } else {
1479 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1480 << "\n");
1481 auto Undef = UndefValue::get(V->getType());
1482 auto SDV =
1483 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1484 DAG.AddDbgValue(SDV, false);
1485 }
1486 }
1487 DDIV.clear();
1488}
1489
1491 DanglingDebugInfo &DDI) {
1492 // TODO: For the variadic implementation, instead of only checking the fail
1493 // state of `handleDebugValue`, we need know specifically which values were
1494 // invalid, so that we attempt to salvage only those values when processing
1495 // a DIArgList.
1496 const Value *OrigV = V;
1497 DILocalVariable *Var = DDI.getVariable();
1498 DIExpression *Expr = DDI.getExpression();
1499 DebugLoc DL = DDI.getDebugLoc();
1500 unsigned SDOrder = DDI.getSDNodeOrder();
1501
1502 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1503 // that DW_OP_stack_value is desired.
1504 bool StackValue = true;
1505
1506 // Can this Value can be encoded without any further work?
1507 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1508 return;
1509
1510 // Attempt to salvage back through as many instructions as possible. Bail if
1511 // a non-instruction is seen, such as a constant expression or global
1512 // variable. FIXME: Further work could recover those too.
1513 while (isa<Instruction>(V)) {
1514 const Instruction &VAsInst = *cast<const Instruction>(V);
1515 // Temporary "0", awaiting real implementation.
1517 SmallVector<Value *, 4> AdditionalValues;
1518 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1519 Expr->getNumLocationOperands(), Ops,
1520 AdditionalValues);
1521 // If we cannot salvage any further, and haven't yet found a suitable debug
1522 // expression, bail out.
1523 if (!V)
1524 break;
1525
1526 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1527 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1528 // here for variadic dbg_values, remove that condition.
1529 if (!AdditionalValues.empty())
1530 break;
1531
1532 // New value and expr now represent this debuginfo.
1533 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1534
1535 // Some kind of simplification occurred: check whether the operand of the
1536 // salvaged debug expression can be encoded in this DAG.
1537 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1538 LLVM_DEBUG(
1539 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1540 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1541 return;
1542 }
1543 }
1544
1545 // This was the final opportunity to salvage this debug information, and it
1546 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1547 // any earlier variable location.
1548 assert(OrigV && "V shouldn't be null");
1549 auto *Undef = UndefValue::get(OrigV->getType());
1550 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1551 DAG.AddDbgValue(SDV, false);
1552 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1553 << printDDI(OrigV, DDI) << "\n");
1554}
1555
1557 DIExpression *Expr,
1558 DebugLoc DbgLoc,
1559 unsigned Order) {
1563 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1564 /*IsVariadic*/ false);
1565}
1566
1568 DILocalVariable *Var,
1569 DIExpression *Expr, DebugLoc DbgLoc,
1570 unsigned Order, bool IsVariadic) {
1571 if (Values.empty())
1572 return true;
1573
1574 // Filter EntryValue locations out early.
1575 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1576 return true;
1577
1578 SmallVector<SDDbgOperand> LocationOps;
1579 SmallVector<SDNode *> Dependencies;
1580 for (const Value *V : Values) {
1581 // Constant value.
1582 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1583 isa<ConstantPointerNull>(V)) {
1584 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1585 continue;
1586 }
1587
1588 // Look through IntToPtr constants.
1589 if (auto *CE = dyn_cast<ConstantExpr>(V))
1590 if (CE->getOpcode() == Instruction::IntToPtr) {
1591 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1592 continue;
1593 }
1594
1595 // If the Value is a frame index, we can create a FrameIndex debug value
1596 // without relying on the DAG at all.
1597 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1598 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1599 if (SI != FuncInfo.StaticAllocaMap.end()) {
1600 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1601 continue;
1602 }
1603 }
1604
1605 // Do not use getValue() in here; we don't want to generate code at
1606 // this point if it hasn't been done yet.
1607 SDValue N = NodeMap[V];
1608 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1609 N = UnusedArgNodeMap[V];
1610 if (N.getNode()) {
1611 // Only emit func arg dbg value for non-variadic dbg.values for now.
1612 if (!IsVariadic &&
1613 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1614 FuncArgumentDbgValueKind::Value, N))
1615 return true;
1616 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1617 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1618 // describe stack slot locations.
1619 //
1620 // Consider "int x = 0; int *px = &x;". There are two kinds of
1621 // interesting debug values here after optimization:
1622 //
1623 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1624 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1625 //
1626 // Both describe the direct values of their associated variables.
1627 Dependencies.push_back(N.getNode());
1628 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1629 continue;
1630 }
1631 LocationOps.emplace_back(
1632 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1633 continue;
1634 }
1635
1637 // Special rules apply for the first dbg.values of parameter variables in a
1638 // function. Identify them by the fact they reference Argument Values, that
1639 // they're parameters, and they are parameters of the current function. We
1640 // need to let them dangle until they get an SDNode.
1641 bool IsParamOfFunc =
1642 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1643 if (IsParamOfFunc)
1644 return false;
1645
1646 // The value is not used in this block yet (or it would have an SDNode).
1647 // We still want the value to appear for the user if possible -- if it has
1648 // an associated VReg, we can refer to that instead.
1649 auto VMI = FuncInfo.ValueMap.find(V);
1650 if (VMI != FuncInfo.ValueMap.end()) {
1651 unsigned Reg = VMI->second;
1652 // If this is a PHI node, it may be split up into several MI PHI nodes
1653 // (in FunctionLoweringInfo::set).
1654 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1655 V->getType(), std::nullopt);
1656 if (RFV.occupiesMultipleRegs()) {
1657 // FIXME: We could potentially support variadic dbg_values here.
1658 if (IsVariadic)
1659 return false;
1660 unsigned Offset = 0;
1661 unsigned BitsToDescribe = 0;
1662 if (auto VarSize = Var->getSizeInBits())
1663 BitsToDescribe = *VarSize;
1664 if (auto Fragment = Expr->getFragmentInfo())
1665 BitsToDescribe = Fragment->SizeInBits;
1666 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1667 // Bail out if all bits are described already.
1668 if (Offset >= BitsToDescribe)
1669 break;
1670 // TODO: handle scalable vectors.
1671 unsigned RegisterSize = RegAndSize.second;
1672 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1673 ? BitsToDescribe - Offset
1674 : RegisterSize;
1675 auto FragmentExpr = DIExpression::createFragmentExpression(
1676 Expr, Offset, FragmentSize);
1677 if (!FragmentExpr)
1678 continue;
1680 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
1681 DAG.AddDbgValue(SDV, false);
1682 Offset += RegisterSize;
1683 }
1684 return true;
1685 }
1686 // We can use simple vreg locations for variadic dbg_values as well.
1687 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1688 continue;
1689 }
1690 // We failed to create a SDDbgOperand for V.
1691 return false;
1692 }
1693
1694 // We have created a SDDbgOperand for each Value in Values.
1695 // Should use Order instead of SDNodeOrder?
1696 assert(!LocationOps.empty());
1697 SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1698 /*IsIndirect=*/false, DbgLoc,
1699 SDNodeOrder, IsVariadic);
1700 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1701 return true;
1702}
1703
1705 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1706 for (auto &Pair : DanglingDebugInfoMap)
1707 for (auto &DDI : Pair.second)
1708 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1710}
1711
1712/// getCopyFromRegs - If there was virtual register allocated for the value V
1713/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1716 SDValue Result;
1717
1718 if (It != FuncInfo.ValueMap.end()) {
1719 Register InReg = It->second;
1720
1722 DAG.getDataLayout(), InReg, Ty,
1723 std::nullopt); // This is not an ABI copy.
1724 SDValue Chain = DAG.getEntryNode();
1725 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1726 V);
1727 resolveDanglingDebugInfo(V, Result);
1728 }
1729
1730 return Result;
1731}
1732
1733/// getValue - Return an SDValue for the given Value.
1735 // If we already have an SDValue for this value, use it. It's important
1736 // to do this first, so that we don't create a CopyFromReg if we already
1737 // have a regular SDValue.
1738 SDValue &N = NodeMap[V];
1739 if (N.getNode()) return N;
1740
1741 // If there's a virtual register allocated and initialized for this
1742 // value, use it.
1743 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1744 return copyFromReg;
1745
1746 // Otherwise create a new SDValue and remember it.
1747 SDValue Val = getValueImpl(V);
1748 NodeMap[V] = Val;
1750 return Val;
1751}
1752
1753/// getNonRegisterValue - Return an SDValue for the given Value, but
1754/// don't look in FuncInfo.ValueMap for a virtual register.
1756 // If we already have an SDValue for this value, use it.
1757 SDValue &N = NodeMap[V];
1758 if (N.getNode()) {
1759 if (isIntOrFPConstant(N)) {
1760 // Remove the debug location from the node as the node is about to be used
1761 // in a location which may differ from the original debug location. This
1762 // is relevant to Constant and ConstantFP nodes because they can appear
1763 // as constant expressions inside PHI nodes.
1764 N->setDebugLoc(DebugLoc());
1765 }
1766 return N;
1767 }
1768
1769 // Otherwise create a new SDValue and remember it.
1770 SDValue Val = getValueImpl(V);
1771 NodeMap[V] = Val;
1773 return Val;
1774}
1775
1776/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1777/// Create an SDValue for the given value.
1780
1781 if (const Constant *C = dyn_cast<Constant>(V)) {
1782 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1783
1784 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1785 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1786
1787 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1788 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1789
1790 if (isa<ConstantPointerNull>(C)) {
1791 unsigned AS = V->getType()->getPointerAddressSpace();
1792 return DAG.getConstant(0, getCurSDLoc(),
1793 TLI.getPointerTy(DAG.getDataLayout(), AS));
1794 }
1795
1796 if (match(C, m_VScale()))
1797 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1798
1799 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1800 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1801
1802 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1803 return DAG.getUNDEF(VT);
1804
1805 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1806 visit(CE->getOpcode(), *CE);
1807 SDValue N1 = NodeMap[V];
1808 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1809 return N1;
1810 }
1811
1812 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1814 for (const Use &U : C->operands()) {
1815 SDNode *Val = getValue(U).getNode();
1816 // If the operand is an empty aggregate, there are no values.
1817 if (!Val) continue;
1818 // Add each leaf value from the operand to the Constants list
1819 // to form a flattened list of all the values.
1820 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1821 Constants.push_back(SDValue(Val, i));
1822 }
1823
1825 }
1826
1827 if (const ConstantDataSequential *CDS =
1828 dyn_cast<ConstantDataSequential>(C)) {
1830 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1831 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1832 // Add each leaf value from the operand to the Constants list
1833 // to form a flattened list of all the values.
1834 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1835 Ops.push_back(SDValue(Val, i));
1836 }
1837
1838 if (isa<ArrayType>(CDS->getType()))
1839 return DAG.getMergeValues(Ops, getCurSDLoc());
1840 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1841 }
1842
1843 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1844 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1845 "Unknown struct or array constant!");
1846
1847 SmallVector<EVT, 4> ValueVTs;
1848 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1849 unsigned NumElts = ValueVTs.size();
1850 if (NumElts == 0)
1851 return SDValue(); // empty struct
1853 for (unsigned i = 0; i != NumElts; ++i) {
1854 EVT EltVT = ValueVTs[i];
1855 if (isa<UndefValue>(C))
1856 Constants[i] = DAG.getUNDEF(EltVT);
1857 else if (EltVT.isFloatingPoint())
1858 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1859 else
1860 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1861 }
1862
1864 }
1865
1866 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1867 return DAG.getBlockAddress(BA, VT);
1868
1869 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1870 return getValue(Equiv->getGlobalValue());
1871
1872 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1873 return getValue(NC->getGlobalValue());
1874
1875 if (VT == MVT::aarch64svcount) {
1876 assert(C->isNullValue() && "Can only zero this target type!");
1877 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1878 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1879 }
1880
1881 VectorType *VecTy = cast<VectorType>(V->getType());
1882
1883 // Now that we know the number and type of the elements, get that number of
1884 // elements into the Ops array based on what kind of constant it is.
1885 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1887 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1888 for (unsigned i = 0; i != NumElements; ++i)
1889 Ops.push_back(getValue(CV->getOperand(i)));
1890
1891 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1892 }
1893
1894 if (isa<ConstantAggregateZero>(C)) {
1895 EVT EltVT =
1897
1898 SDValue Op;
1899 if (EltVT.isFloatingPoint())
1900 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1901 else
1902 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1903
1904 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1905 }
1906
1907 llvm_unreachable("Unknown vector constant");
1908 }
1909
1910 // If this is a static alloca, generate it as the frameindex instead of
1911 // computation.
1912 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1914 FuncInfo.StaticAllocaMap.find(AI);
1915 if (SI != FuncInfo.StaticAllocaMap.end())
1916 return DAG.getFrameIndex(
1917 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1918 }
1919
1920 // If this is an instruction which fast-isel has deferred, select it now.
1921 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1923
1924 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1925 Inst->getType(), std::nullopt);
1926 SDValue Chain = DAG.getEntryNode();
1927 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1928 }
1929
1930 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1931 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1932
1933 if (const auto *BB = dyn_cast<BasicBlock>(V))
1934 return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1935
1936 llvm_unreachable("Can't get register for value!");
1937}
1938
1939void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1941 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1942 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1943 bool IsSEH = isAsynchronousEHPersonality(Pers);
1944 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1945 if (!IsSEH)
1946 CatchPadMBB->setIsEHScopeEntry();
1947 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1948 if (IsMSVCCXX || IsCoreCLR)
1949 CatchPadMBB->setIsEHFuncletEntry();
1950}
1951
1952void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1953 // Update machine-CFG edge.
1954 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1955 FuncInfo.MBB->addSuccessor(TargetMBB);
1956 TargetMBB->setIsEHCatchretTarget(true);
1958
1960 bool IsSEH = isAsynchronousEHPersonality(Pers);
1961 if (IsSEH) {
1962 // If this is not a fall-through branch or optimizations are switched off,
1963 // emit the branch.
1964 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1966 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1967 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1968 return;
1969 }
1970
1971 // Figure out the funclet membership for the catchret's successor.
1972 // This will be used by the FuncletLayout pass to determine how to order the
1973 // BB's.
1974 // A 'catchret' returns to the outer scope's color.
1975 Value *ParentPad = I.getCatchSwitchParentPad();
1976 const BasicBlock *SuccessorColor;
1977 if (isa<ConstantTokenNone>(ParentPad))
1978 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1979 else
1980 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1981 assert(SuccessorColor && "No parent funclet for catchret!");
1982 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1983 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1984
1985 // Create the terminator node.
1987 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1988 DAG.getBasicBlock(SuccessorColorMBB));
1989 DAG.setRoot(Ret);
1990}
1991
1992void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1993 // Don't emit any special code for the cleanuppad instruction. It just marks
1994 // the start of an EH scope/funclet.
1997 if (Pers != EHPersonality::Wasm_CXX) {
2000 }
2001}
2002
2003// In wasm EH, even though a catchpad may not catch an exception if a tag does
2004// not match, it is OK to add only the first unwind destination catchpad to the
2005// successors, because there will be at least one invoke instruction within the
2006// catch scope that points to the next unwind destination, if one exists, so
2007// CFGSort cannot mess up with BB sorting order.
2008// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
2009// call within them, and catchpads only consisting of 'catch (...)' have a
2010// '__cxa_end_catch' call within them, both of which generate invokes in case
2011// the next unwind destination exists, i.e., the next unwind destination is not
2012// the caller.)
2013//
2014// Having at most one EH pad successor is also simpler and helps later
2015// transformations.
2016//
2017// For example,
2018// current:
2019// invoke void @foo to ... unwind label %catch.dispatch
2020// catch.dispatch:
2021// %0 = catchswitch within ... [label %catch.start] unwind label %next
2022// catch.start:
2023// ...
2024// ... in this BB or some other child BB dominated by this BB there will be an
2025// invoke that points to 'next' BB as an unwind destination
2026//
2027// next: ; We don't need to add this to 'current' BB's successor
2028// ...
2030 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2031 BranchProbability Prob,
2032 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2033 &UnwindDests) {
2034 while (EHPadBB) {
2035 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2036 if (isa<CleanupPadInst>(Pad)) {
2037 // Stop on cleanup pads.
2038 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2039 UnwindDests.back().first->setIsEHScopeEntry();
2040 break;
2041 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2042 // Add the catchpad handlers to the possible destinations. We don't
2043 // continue to the unwind destination of the catchswitch for wasm.
2044 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2045 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
2046 UnwindDests.back().first->setIsEHScopeEntry();
2047 }
2048 break;
2049 } else {
2050 continue;
2051 }
2052 }
2053}
2054
2055/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2056/// many places it could ultimately go. In the IR, we have a single unwind
2057/// destination, but in the machine CFG, we enumerate all the possible blocks.
2058/// This function skips over imaginary basic blocks that hold catchswitch
2059/// instructions, and finds all the "real" machine
2060/// basic block destinations. As those destinations may not be successors of
2061/// EHPadBB, here we also calculate the edge probability to those destinations.
2062/// The passed-in Prob is the edge probability to EHPadBB.
2064 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2065 BranchProbability Prob,
2066 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2067 &UnwindDests) {
2068 EHPersonality Personality =
2070 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2071 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2072 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2073 bool IsSEH = isAsynchronousEHPersonality(Personality);
2074
2075 if (IsWasmCXX) {
2076 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
2077 assert(UnwindDests.size() <= 1 &&
2078 "There should be at most one unwind destination for wasm");
2079 return;
2080 }
2081
2082 while (EHPadBB) {
2083 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2084 BasicBlock *NewEHPadBB = nullptr;
2085 if (isa<LandingPadInst>(Pad)) {
2086 // Stop on landingpads. They are not funclets.
2087 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2088 break;
2089 } else if (isa<CleanupPadInst>(Pad)) {
2090 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2091 // personalities.
2092 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
2093 UnwindDests.back().first->setIsEHScopeEntry();
2094 UnwindDests.back().first->setIsEHFuncletEntry();
2095 break;
2096 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2097 // Add the catchpad handlers to the possible destinations.
2098 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2099 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
2100 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2101 if (IsMSVCCXX || IsCoreCLR)
2102 UnwindDests.back().first->setIsEHFuncletEntry();
2103 if (!IsSEH)
2104 UnwindDests.back().first->setIsEHScopeEntry();
2105 }
2106 NewEHPadBB = CatchSwitch->getUnwindDest();
2107 } else {
2108 continue;
2109 }
2110
2111 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2112 if (BPI && NewEHPadBB)
2113 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2114 EHPadBB = NewEHPadBB;
2115 }
2116}
2117
2118void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2119 // Update successor info.
2121 auto UnwindDest = I.getUnwindDest();
2123 BranchProbability UnwindDestProb =
2124 (BPI && UnwindDest)
2125 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2127 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2128 for (auto &UnwindDest : UnwindDests) {
2129 UnwindDest.first->setIsEHPad();
2130 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2131 }
2133
2134 // Create the terminator node.
2135 SDValue Ret =
2137 DAG.setRoot(Ret);
2138}
2139
2140void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2141 report_fatal_error("visitCatchSwitch not yet implemented!");
2142}
2143
2144void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2146 auto &DL = DAG.getDataLayout();
2147 SDValue Chain = getControlRoot();
2150
2151 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2152 // lower
2153 //
2154 // %val = call <ty> @llvm.experimental.deoptimize()
2155 // ret <ty> %val
2156 //
2157 // differently.
2158 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2160 return;
2161 }
2162
2163 if (!FuncInfo.CanLowerReturn) {
2164 unsigned DemoteReg = FuncInfo.DemoteRegister;
2165 const Function *F = I.getParent()->getParent();
2166
2167 // Emit a store of the return value through the virtual register.
2168 // Leave Outs empty so that LowerReturn won't try to load return
2169 // registers the usual way.
2170 SmallVector<EVT, 1> PtrValueVTs;
2171 ComputeValueVTs(TLI, DL,
2172 PointerType::get(F->getContext(),
2174 PtrValueVTs);
2175
2176 SDValue RetPtr =
2177 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2178 SDValue RetOp = getValue(I.getOperand(0));
2179
2180 SmallVector<EVT, 4> ValueVTs, MemVTs;
2182 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2183 &Offsets, 0);
2184 unsigned NumValues = ValueVTs.size();
2185
2186 SmallVector<SDValue, 4> Chains(NumValues);
2187 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2188 for (unsigned i = 0; i != NumValues; ++i) {
2189 // An aggregate return value cannot wrap around the address space, so
2190 // offsets to its parts don't wrap either.
2192 TypeSize::getFixed(Offsets[i]));
2193
2194 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2195 if (MemVTs[i] != ValueVTs[i])
2196 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2197 Chains[i] = DAG.getStore(
2198 Chain, getCurSDLoc(), Val,
2199 // FIXME: better loc info would be nice.
2201 commonAlignment(BaseAlign, Offsets[i]));
2202 }
2203
2205 MVT::Other, Chains);
2206 } else if (I.getNumOperands() != 0) {
2207 SmallVector<EVT, 4> ValueVTs;
2208 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2209 unsigned NumValues = ValueVTs.size();
2210 if (NumValues) {
2211 SDValue RetOp = getValue(I.getOperand(0));
2212
2213 const Function *F = I.getParent()->getParent();
2214
2215 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2216 I.getOperand(0)->getType(), F->getCallingConv(),
2217 /*IsVarArg*/ false, DL);
2218
2219 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2220 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2221 ExtendKind = ISD::SIGN_EXTEND;
2222 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2223 ExtendKind = ISD::ZERO_EXTEND;
2224
2225 LLVMContext &Context = F->getContext();
2226 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2227
2228 for (unsigned j = 0; j != NumValues; ++j) {
2229 EVT VT = ValueVTs[j];
2230
2231 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2232 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2233
2234 CallingConv::ID CC = F->getCallingConv();
2235
2236 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2237 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2238 SmallVector<SDValue, 4> Parts(NumParts);
2240 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2241 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2242
2243 // 'inreg' on function refers to return value
2245 if (RetInReg)
2246 Flags.setInReg();
2247
2248 if (I.getOperand(0)->getType()->isPointerTy()) {
2249 Flags.setPointer();
2250 Flags.setPointerAddrSpace(
2251 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2252 }
2253
2254 if (NeedsRegBlock) {
2255 Flags.setInConsecutiveRegs();
2256 if (j == NumValues - 1)
2257 Flags.setInConsecutiveRegsLast();
2258 }
2259
2260 // Propagate extension type if any
2261 if (ExtendKind == ISD::SIGN_EXTEND)
2262 Flags.setSExt();
2263 else if (ExtendKind == ISD::ZERO_EXTEND)
2264 Flags.setZExt();
2265
2266 for (unsigned i = 0; i < NumParts; ++i) {
2267 Outs.push_back(ISD::OutputArg(Flags,
2268 Parts[i].getValueType().getSimpleVT(),
2269 VT, /*isfixed=*/true, 0, 0));
2270 OutVals.push_back(Parts[i]);
2271 }
2272 }
2273 }
2274 }
2275
2276 // Push in swifterror virtual register as the last element of Outs. This makes
2277 // sure swifterror virtual register will be returned in the swifterror
2278 // physical register.
2279 const Function *F = I.getParent()->getParent();
2280 if (TLI.supportSwiftError() &&
2281 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2282 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2284 Flags.setSwiftError();
2286 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2287 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2288 // Create SDNode for the swifterror virtual register.
2289 OutVals.push_back(
2292 EVT(TLI.getPointerTy(DL))));
2293 }
2294
2295 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2296 CallingConv::ID CallConv =
2299 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2300
2301 // Verify that the target's LowerReturn behaved as expected.
2302 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2303 "LowerReturn didn't return a valid chain!");
2304
2305 // Update the DAG with the new chain value resulting from return lowering.
2306 DAG.setRoot(Chain);
2307}
2308
2309/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2310/// created for it, emit nodes to copy the value into the virtual
2311/// registers.
2313 // Skip empty types
2314 if (V->getType()->isEmptyTy())
2315 return;
2316
2318 if (VMI != FuncInfo.ValueMap.end()) {
2319 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2320 "Unused value assigned virtual registers!");
2321 CopyValueToVirtualRegister(V, VMI->second);
2322 }
2323}
2324
2325/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2326/// the current basic block, add it to ValueMap now so that we'll get a
2327/// CopyTo/FromReg.
2329 // No need to export constants.
2330 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2331
2332 // Already exported?
2333 if (FuncInfo.isExportedInst(V)) return;
2334
2337}
2338
2340 const BasicBlock *FromBB) {
2341 // The operands of the setcc have to be in this block. We don't know
2342 // how to export them from some other block.
2343 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2344 // Can export from current BB.
2345 if (VI->getParent() == FromBB)
2346 return true;
2347
2348 // Is already exported, noop.
2349 return FuncInfo.isExportedInst(V);
2350 }
2351
2352 // If this is an argument, we can export it if the BB is the entry block or
2353 // if it is already exported.
2354 if (isa<Argument>(V)) {
2355 if (FromBB->isEntryBlock())
2356 return true;
2357
2358 // Otherwise, can only export this if it is already exported.
2359 return FuncInfo.isExportedInst(V);
2360 }
2361
2362 // Otherwise, constants can always be exported.
2363 return true;
2364}
2365
2366/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2368SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2369 const MachineBasicBlock *Dst) const {
2371 const BasicBlock *SrcBB = Src->getBasicBlock();
2372 const BasicBlock *DstBB = Dst->getBasicBlock();
2373 if (!BPI) {
2374 // If BPI is not available, set the default probability as 1 / N, where N is
2375 // the number of successors.
2376 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2377 return BranchProbability(1, SuccSize);
2378 }
2379 return BPI->getEdgeProbability(SrcBB, DstBB);
2380}
2381
2382void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2383 MachineBasicBlock *Dst,
2384 BranchProbability Prob) {
2385 if (!FuncInfo.BPI)
2386 Src->addSuccessorWithoutProb(Dst);
2387 else {
2388 if (Prob.isUnknown())
2389 Prob = getEdgeProbability(Src, Dst);
2390 Src->addSuccessor(Dst, Prob);
2391 }
2392}
2393
2394static bool InBlock(const Value *V, const BasicBlock *BB) {
2395 if (const Instruction *I = dyn_cast<Instruction>(V))
2396 return I->getParent() == BB;
2397 return true;
2398}
2399
2400/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2401/// This function emits a branch and is used at the leaves of an OR or an
2402/// AND operator tree.
2403void
2406 MachineBasicBlock *FBB,
2407 MachineBasicBlock *CurBB,
2408 MachineBasicBlock *SwitchBB,
2409 BranchProbability TProb,
2410 BranchProbability FProb,
2411 bool InvertCond) {
2412 const BasicBlock *BB = CurBB->getBasicBlock();
2413
2414 // If the leaf of the tree is a comparison, merge the condition into
2415 // the caseblock.
2416 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2417 // The operands of the cmp have to be in this block. We don't know
2418 // how to export them from some other block. If this is the first block
2419 // of the sequence, no exporting is needed.
2420 if (CurBB == SwitchBB ||
2421 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2422 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2423 ISD::CondCode Condition;
2424 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2425 ICmpInst::Predicate Pred =
2426 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2427 Condition = getICmpCondCode(Pred);
2428 } else {
2429 const FCmpInst *FC = cast<FCmpInst>(Cond);
2430 FCmpInst::Predicate Pred =
2431 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2432 Condition = getFCmpCondCode(Pred);
2433 if (TM.Options.NoNaNsFPMath)
2434 Condition = getFCmpCodeWithoutNaN(Condition);
2435 }
2436
2437 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2438 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2439 SL->SwitchCases.push_back(CB);
2440 return;
2441 }
2442 }
2443
2444 // Create a CaseBlock record representing this branch.
2445 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2447 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2448 SL->SwitchCases.push_back(CB);
2449}
2450
2451// Collect dependencies on V recursively. This is used for the cost analysis in
2452// `shouldKeepJumpConditionsTogether`.
2456 unsigned Depth = 0) {
2457 // Return false if we have an incomplete count.
2459 return false;
2460
2461 auto *I = dyn_cast<Instruction>(V);
2462 if (I == nullptr)
2463 return true;
2464
2465 if (Necessary != nullptr) {
2466 // This instruction is necessary for the other side of the condition so
2467 // don't count it.
2468 if (Necessary->contains(I))
2469 return true;
2470 }
2471
2472 // Already added this dep.
2473 if (!Deps->try_emplace(I, false).second)
2474 return true;
2475
2476 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2477 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2478 Depth + 1))
2479 return false;
2480 return true;
2481}
2482
2484 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,
2485 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2487 if (I.getNumSuccessors() != 2)
2488 return false;
2489
2490 if (!I.isConditional())
2491 return false;
2492
2493 if (Params.BaseCost < 0)
2494 return false;
2495
2496 // Baseline cost.
2497 InstructionCost CostThresh = Params.BaseCost;
2498
2499 BranchProbabilityInfo *BPI = nullptr;
2500 if (Params.LikelyBias || Params.UnlikelyBias)
2501 BPI = FuncInfo.BPI;
2502 if (BPI != nullptr) {
2503 // See if we are either likely to get an early out or compute both lhs/rhs
2504 // of the condition.
2505 BasicBlock *IfFalse = I.getSuccessor(0);
2506 BasicBlock *IfTrue = I.getSuccessor(1);
2507
2508 std::optional<bool> Likely;
2509 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2510 Likely = true;
2511 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2512 Likely = false;
2513
2514 if (Likely) {
2515 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2516 // Its likely we will have to compute both lhs and rhs of condition
2517 CostThresh += Params.LikelyBias;
2518 else {
2519 if (Params.UnlikelyBias < 0)
2520 return false;
2521 // Its likely we will get an early out.
2522 CostThresh -= Params.UnlikelyBias;
2523 }
2524 }
2525 }
2526
2527 if (CostThresh <= 0)
2528 return false;
2529
2530 // Collect "all" instructions that lhs condition is dependent on.
2531 // Use map for stable iteration (to avoid non-determanism of iteration of
2532 // SmallPtrSet). The `bool` value is just a dummy.
2534 collectInstructionDeps(&LhsDeps, Lhs);
2535 // Collect "all" instructions that rhs condition is dependent on AND are
2536 // dependencies of lhs. This gives us an estimate on which instructions we
2537 // stand to save by splitting the condition.
2538 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2539 return false;
2540 // Add the compare instruction itself unless its a dependency on the LHS.
2541 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2542 if (!LhsDeps.contains(RhsI))
2543 RhsDeps.try_emplace(RhsI, false);
2544
2545 const auto &TLI = DAG.getTargetLoweringInfo();
2546 const auto &TTI =
2547 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
2548
2549 InstructionCost CostOfIncluding = 0;
2550 // See if this instruction will need to computed independently of whether RHS
2551 // is.
2552 Value *BrCond = I.getCondition();
2553 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2554 for (const auto *U : Ins->users()) {
2555 // If user is independent of RHS calculation we don't need to count it.
2556 if (auto *UIns = dyn_cast<Instruction>(U))
2557 if (UIns != BrCond && !RhsDeps.contains(UIns))
2558 return false;
2559 }
2560 return true;
2561 };
2562
2563 // Prune instructions from RHS Deps that are dependencies of unrelated
2564 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2565 // arbitrary and just meant to cap the how much time we spend in the pruning
2566 // loop. Its highly unlikely to come into affect.
2567 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2568 // Stop after a certain point. No incorrectness from including too many
2569 // instructions.
2570 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2571 const Instruction *ToDrop = nullptr;
2572 for (const auto &InsPair : RhsDeps) {
2573 if (!ShouldCountInsn(InsPair.first)) {
2574 ToDrop = InsPair.first;
2575 break;
2576 }
2577 }
2578 if (ToDrop == nullptr)
2579 break;
2580 RhsDeps.erase(ToDrop);
2581 }
2582
2583 for (const auto &InsPair : RhsDeps) {
2584 // Finally accumulate latency that we can only attribute to computing the
2585 // RHS condition. Use latency because we are essentially trying to calculate
2586 // the cost of the dependency chain.
2587 // Possible TODO: We could try to estimate ILP and make this more precise.
2588 CostOfIncluding +=
2590
2591 if (CostOfIncluding > CostThresh)
2592 return false;
2593 }
2594 return true;
2595}
2596
2599 MachineBasicBlock *FBB,
2600 MachineBasicBlock *CurBB,
2601 MachineBasicBlock *SwitchBB,
2603 BranchProbability TProb,
2604 BranchProbability FProb,
2605 bool InvertCond) {
2606 // Skip over not part of the tree and remember to invert op and operands at
2607 // next level.
2608 Value *NotCond;
2609 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2610 InBlock(NotCond, CurBB->getBasicBlock())) {
2611 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2612 !InvertCond);
2613 return;
2614 }
2615
2616 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2617 const Value *BOpOp0, *BOpOp1;
2618 // Compute the effective opcode for Cond, taking into account whether it needs
2619 // to be inverted, e.g.
2620 // and (not (or A, B)), C
2621 // gets lowered as
2622 // and (and (not A, not B), C)
2624 if (BOp) {
2625 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2626 ? Instruction::And
2627 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2628 ? Instruction::Or
2630 if (InvertCond) {
2631 if (BOpc == Instruction::And)
2632 BOpc = Instruction::Or;
2633 else if (BOpc == Instruction::Or)
2634 BOpc = Instruction::And;
2635 }
2636 }
2637
2638 // If this node is not part of the or/and tree, emit it as a branch.
2639 // Note that all nodes in the tree should have same opcode.
2640 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2641 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2642 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2643 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2644 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2645 TProb, FProb, InvertCond);
2646 return;
2647 }
2648
2649 // Create TmpBB after CurBB.
2650 MachineFunction::iterator BBI(CurBB);
2653 CurBB->getParent()->insert(++BBI, TmpBB);
2654
2655 if (Opc == Instruction::Or) {
2656 // Codegen X | Y as:
2657 // BB1:
2658 // jmp_if_X TBB
2659 // jmp TmpBB
2660 // TmpBB:
2661 // jmp_if_Y TBB
2662 // jmp FBB
2663 //
2664
2665 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2666 // The requirement is that
2667 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2668 // = TrueProb for original BB.
2669 // Assuming the original probabilities are A and B, one choice is to set
2670 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2671 // A/(1+B) and 2B/(1+B). This choice assumes that
2672 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2673 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2674 // TmpBB, but the math is more complicated.
2675
2676 auto NewTrueProb = TProb / 2;
2677 auto NewFalseProb = TProb / 2 + FProb;
2678 // Emit the LHS condition.
2679 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2680 NewFalseProb, InvertCond);
2681
2682 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2683 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2684 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2685 // Emit the RHS condition into TmpBB.
2686 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2687 Probs[1], InvertCond);
2688 } else {
2689 assert(Opc == Instruction::And && "Unknown merge op!");
2690 // Codegen X & Y as:
2691 // BB1:
2692 // jmp_if_X TmpBB
2693 // jmp FBB
2694 // TmpBB:
2695 // jmp_if_Y TBB
2696 // jmp FBB
2697 //
2698 // This requires creation of TmpBB after CurBB.
2699
2700 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2701 // The requirement is that
2702 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2703 // = FalseProb for original BB.
2704 // Assuming the original probabilities are A and B, one choice is to set
2705 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2706 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2707 // TrueProb for BB1 * FalseProb for TmpBB.
2708
2709 auto NewTrueProb = TProb + FProb / 2;
2710 auto NewFalseProb = FProb / 2;
2711 // Emit the LHS condition.
2712 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2713 NewFalseProb, InvertCond);
2714
2715 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2716 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2717 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2718 // Emit the RHS condition into TmpBB.
2719 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2720 Probs[1], InvertCond);
2721 }
2722}
2723
2724/// If the set of cases should be emitted as a series of branches, return true.
2725/// If we should emit this as a bunch of and/or'd together conditions, return
2726/// false.
2727bool
2728SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2729 if (Cases.size() != 2) return true;
2730
2731 // If this is two comparisons of the same values or'd or and'd together, they
2732 // will get folded into a single comparison, so don't emit two blocks.
2733 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2734 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2735 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2736 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2737 return false;
2738 }
2739
2740 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2741 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2742 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2743 Cases[0].CC == Cases[1].CC &&
2744 isa<Constant>(Cases[0].CmpRHS) &&
2745 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2746 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2747 return false;
2748 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2749 return false;
2750 }
2751
2752 return true;
2753}
2754
2755void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2757
2758 // Update machine-CFG edges.
2759 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2760
2761 if (I.isUnconditional()) {
2762 // Update machine-CFG edges.
2763 BrMBB->addSuccessor(Succ0MBB);
2764
2765 // If this is not a fall-through branch or optimizations are switched off,
2766 // emit the branch.
2767 if (Succ0MBB != NextBlock(BrMBB) ||
2769 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2770 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2771 setValue(&I, Br);
2772 DAG.setRoot(Br);
2773 }
2774
2775 return;
2776 }
2777
2778 // If this condition is one of the special cases we handle, do special stuff
2779 // now.
2780 const Value *CondVal = I.getCondition();
2781 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2782
2783 // If this is a series of conditions that are or'd or and'd together, emit
2784 // this as a sequence of branches instead of setcc's with and/or operations.
2785 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2786 // unpredictable branches, and vector extracts because those jumps are likely
2787 // expensive for any target), this should improve performance.
2788 // For example, instead of something like:
2789 // cmp A, B
2790 // C = seteq
2791 // cmp D, E
2792 // F = setle
2793 // or C, F
2794 // jnz foo
2795 // Emit:
2796 // cmp A, B
2797 // je foo
2798 // cmp D, E
2799 // jle foo
2800 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2801 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2802 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2803 Value *Vec;
2804 const Value *BOp0, *BOp1;
2806 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2807 Opcode = Instruction::And;
2808 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2809 Opcode = Instruction::Or;
2810
2811 if (Opcode &&
2812 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2813 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2815 FuncInfo, I, Opcode, BOp0, BOp1,
2817 Opcode, BOp0, BOp1))) {
2818 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2819 getEdgeProbability(BrMBB, Succ0MBB),
2820 getEdgeProbability(BrMBB, Succ1MBB),
2821 /*InvertCond=*/false);
2822 // If the compares in later blocks need to use values not currently
2823 // exported from this block, export them now. This block should always
2824 // be the first entry.
2825 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2826
2827 // Allow some cases to be rejected.
2828 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2829 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2830 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2831 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2832 }
2833
2834 // Emit the branch for this block.
2835 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2836 SL->SwitchCases.erase(SL->SwitchCases.begin());
2837 return;
2838 }
2839
2840 // Okay, we decided not to do this, remove any inserted MBB's and clear
2841 // SwitchCases.
2842 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2843 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2844
2845 SL->SwitchCases.clear();
2846 }
2847 }
2848
2849 // Create a CaseBlock record representing this branch.
2851 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2852
2853 // Use visitSwitchCase to actually insert the fast branch sequence for this
2854 // cond branch.
2855 visitSwitchCase(CB, BrMBB);
2856}
2857
2858/// visitSwitchCase - Emits the necessary code to represent a single node in
2859/// the binary search tree resulting from lowering a switch instruction.
2861 MachineBasicBlock *SwitchBB) {
2862 SDValue Cond;
2863 SDValue CondLHS = getValue(CB.CmpLHS);
2864 SDLoc dl = CB.DL;
2865
2866 if (CB.CC == ISD::SETTRUE) {
2867 // Branch or fall through to TrueBB.
2868 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2869 SwitchBB->normalizeSuccProbs();
2870 if (CB.TrueBB != NextBlock(SwitchBB)) {
2871 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2872 DAG.getBasicBlock(CB.TrueBB)));
2873 }
2874 return;
2875 }
2876
2877 auto &TLI = DAG.getTargetLoweringInfo();
2878 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2879
2880 // Build the setcc now.
2881 if (!CB.CmpMHS) {
2882 // Fold "(X == true)" to X and "(X == false)" to !X to
2883 // handle common cases produced by branch lowering.
2884 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2885 CB.CC == ISD::SETEQ)
2886 Cond = CondLHS;
2887 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2888 CB.CC == ISD::SETEQ) {
2889 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2890 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2891 } else {
2892 SDValue CondRHS = getValue(CB.CmpRHS);
2893
2894 // If a pointer's DAG type is larger than its memory type then the DAG
2895 // values are zero-extended. This breaks signed comparisons so truncate
2896 // back to the underlying type before doing the compare.
2897 if (CondLHS.getValueType() != MemVT) {
2898 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2899 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2900 }
2901 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2902 }
2903 } else {
2904 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2905
2906 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2907 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2908
2909 SDValue CmpOp = getValue(CB.CmpMHS);
2910 EVT VT = CmpOp.getValueType();
2911
2912 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2913 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2914 ISD::SETLE);
2915 } else {
2916 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2917 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2918 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2919 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2920 }
2921 }
2922
2923 // Update successor info
2924 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2925 // TrueBB and FalseBB are always different unless the incoming IR is
2926 // degenerate. This only happens when running llc on weird IR.
2927 if (CB.TrueBB != CB.FalseBB)
2928 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2929 SwitchBB->normalizeSuccProbs();
2930
2931 // If the lhs block is the next block, invert the condition so that we can
2932 // fall through to the lhs instead of the rhs block.
2933 if (CB.TrueBB == NextBlock(SwitchBB)) {
2934 std::swap(CB.TrueBB, CB.FalseBB);
2935 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2936 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2937 }
2938
2939 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2940 MVT::Other, getControlRoot(), Cond,
2942
2943 setValue(CurInst, BrCond);
2944
2945 // Insert the false branch. Do this even if it's a fall through branch,
2946 // this makes it easier to do DAG optimizations which require inverting
2947 // the branch condition.
2948 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2950
2951 DAG.setRoot(BrCond);
2952}
2953
2954/// visitJumpTable - Emit JumpTable node in the current MBB
2956 // Emit the code for the jump table
2957 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2958 assert(JT.Reg != -1U && "Should lower JT Header first!");
2960 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
2961 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2962 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2963 Index.getValue(1), Table, Index);
2964 DAG.setRoot(BrJumpTable);
2965}
2966
2967/// visitJumpTableHeader - This function emits necessary code to produce index
2968/// in the JumpTable from switch case.
2970 JumpTableHeader &JTH,
2971 MachineBasicBlock *SwitchBB) {
2972 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2973 const SDLoc &dl = *JT.SL;
2974
2975 // Subtract the lowest switch case value from the value being switched on.
2976 SDValue SwitchOp = getValue(JTH.SValue);
2977 EVT VT = SwitchOp.getValueType();
2978 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2979 DAG.getConstant(JTH.First, dl, VT));
2980
2981 // The SDNode we just created, which holds the value being switched on minus
2982 // the smallest case value, needs to be copied to a virtual register so it
2983 // can be used as an index into the jump table in a subsequent basic block.
2984 // This value may be smaller or larger than the target's pointer type, and
2985 // therefore require extension or truncating.
2987 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2988
2989 unsigned JumpTableReg =
2991 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2992 JumpTableReg, SwitchOp);
2993 JT.Reg = JumpTableReg;
2994
2995 if (!JTH.FallthroughUnreachable) {
2996 // Emit the range check for the jump table, and branch to the default block
2997 // for the switch statement if the value being switched on exceeds the
2998 // largest case in the switch.
2999 SDValue CMP = DAG.getSetCC(
3001 Sub.getValueType()),
3002 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3003
3004 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3005 MVT::Other, CopyTo, CMP,
3006 DAG.getBasicBlock(JT.Default));
3007
3008 // Avoid emitting unnecessary branches to the next block.
3009 if (JT.MBB != NextBlock(SwitchBB))
3010 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3011 DAG.getBasicBlock(JT.MBB));
3012
3013 DAG.setRoot(BrCond);
3014 } else {
3015 // Avoid emitting unnecessary branches to the next block.
3016 if (JT.MBB != NextBlock(SwitchBB))
3017 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3018 DAG.getBasicBlock(JT.MBB)));
3019 else
3020 DAG.setRoot(CopyTo);
3021 }
3022}
3023
3024/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3025/// variable if there exists one.
3027 SDValue &Chain) {
3028 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3029 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3030 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3034 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3035 if (Global) {
3036 MachinePointerInfo MPInfo(Global);
3040 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
3041 DAG.setNodeMemRefs(Node, {MemRef});
3042 }
3043 if (PtrTy != PtrMemTy)
3044 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3045 return SDValue(Node, 0);
3046}
3047
3048/// Codegen a new tail for a stack protector check ParentMBB which has had its
3049/// tail spliced into a stack protector check success bb.
3050///
3051/// For a high level explanation of how this fits into the stack protector
3052/// generation see the comment on the declaration of class
3053/// StackProtectorDescriptor.
3055 MachineBasicBlock *ParentBB) {
3056
3057 // First create the loads to the guard/stack slot for the comparison.
3059 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3060 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3061
3062 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3063 int FI = MFI.getStackProtectorIndex();
3064
3065 SDValue Guard;
3066 SDLoc dl = getCurSDLoc();
3067 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3068 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3069 Align Align =
3070 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
3071
3072 // Generate code to load the content of the guard slot.
3073 SDValue GuardVal = DAG.getLoad(
3074 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3077
3078 if (TLI.useStackGuardXorFP())
3079 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3080
3081 // Retrieve guard check function, nullptr if instrumentation is inlined.
3082 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3083 // The target provides a guard check function to validate the guard value.
3084 // Generate a call to that function with the content of the guard slot as
3085 // argument.
3086 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3087 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3088
3091 Entry.Node = GuardVal;
3092 Entry.Ty = FnTy->getParamType(0);
3093 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3094 Entry.IsInReg = true;
3095 Args.push_back(Entry);
3096
3100 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3101 getValue(GuardCheckFn), std::move(Args));
3102
3103 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3104 DAG.setRoot(Result.second);
3105 return;
3106 }
3107
3108 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3109 // Otherwise, emit a volatile load to retrieve the stack guard value.
3110 SDValue Chain = DAG.getEntryNode();
3111 if (TLI.useLoadStackGuardNode()) {
3112 Guard = getLoadStackGuard(DAG, dl, Chain);
3113 } else {
3114 const Value *IRGuard = TLI.getSDagStackGuard(M);
3115 SDValue GuardPtr = getValue(IRGuard);
3116
3117 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3118 MachinePointerInfo(IRGuard, 0), Align,
3120 }
3121
3122 // Perform the comparison via a getsetcc.
3124 *DAG.getContext(),
3125 Guard.getValueType()),
3126 Guard, GuardVal, ISD::SETNE);
3127
3128 // If the guard/stackslot do not equal, branch to failure MBB.
3129 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3130 MVT::Other, GuardVal.getOperand(0),
3131 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3132 // Otherwise branch to success MBB.
3133 SDValue Br = DAG.getNode(ISD::BR, dl,
3134 MVT::Other, BrCond,
3136
3137 DAG.setRoot(Br);
3138}
3139
3140/// Codegen the failure basic block for a stack protector check.
3141///
3142/// A failure stack protector machine basic block consists simply of a call to
3143/// __stack_chk_fail().
3144///
3145/// For a high level explanation of how this fits into the stack protector
3146/// generation see the comment on the declaration of class
3147/// StackProtectorDescriptor.
3148void
3152 CallOptions.setDiscardResult(true);
3153 SDValue Chain =
3154 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3155 std::nullopt, CallOptions, getCurSDLoc())
3156 .second;
3157 // On PS4/PS5, the "return address" must still be within the calling
3158 // function, even if it's at the very end, so emit an explicit TRAP here.
3159 // Passing 'true' for doesNotReturn above won't generate the trap for us.
3160 if (TM.getTargetTriple().isPS())
3161 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3162 // WebAssembly needs an unreachable instruction after a non-returning call,
3163 // because the function return type can be different from __stack_chk_fail's
3164 // return type (void).
3165 if (TM.getTargetTriple().isWasm())
3166 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3167
3168 DAG.setRoot(Chain);
3169}
3170
3171/// visitBitTestHeader - This function emits necessary code to produce value
3172/// suitable for "bit tests"
3174 MachineBasicBlock *SwitchBB) {
3175 SDLoc dl = getCurSDLoc();
3176
3177 // Subtract the minimum value.
3178 SDValue SwitchOp = getValue(B.SValue);
3179 EVT VT = SwitchOp.getValueType();
3180 SDValue RangeSub =
3181 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3182
3183 // Determine the type of the test operands.
3185 bool UsePtrType = false;
3186 if (!TLI.isTypeLegal(VT)) {
3187 UsePtrType = true;
3188 } else {
3189 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
3190 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
3191 // Switch table case range are encoded into series of masks.
3192 // Just use pointer type, it's guaranteed to fit.
3193 UsePtrType = true;
3194 break;
3195 }
3196 }
3197 SDValue Sub = RangeSub;
3198 if (UsePtrType) {
3199 VT = TLI.getPointerTy(DAG.getDataLayout());
3200 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3201 }
3202
3203 B.RegVT = VT.getSimpleVT();
3204 B.Reg = FuncInfo.CreateReg(B.RegVT);
3205 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3206
3207 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3208
3209 if (!B.FallthroughUnreachable)
3210 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3211 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3212 SwitchBB->normalizeSuccProbs();
3213
3214 SDValue Root = CopyTo;
3215 if (!B.FallthroughUnreachable) {
3216 // Conditional branch to the default block.
3217 SDValue RangeCmp = DAG.getSetCC(dl,
3219 RangeSub.getValueType()),
3220 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3221 ISD::SETUGT);
3222
3223 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3224 DAG.getBasicBlock(B.Default));
3225 }
3226
3227 // Avoid emitting unnecessary branches to the next block.
3228 if (MBB != NextBlock(SwitchBB))
3229 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3230
3231 DAG.setRoot(Root);
3232}
3233
3234/// visitBitTestCase - this function produces one "bit test"
3236 MachineBasicBlock* NextMBB,
3237 BranchProbability BranchProbToNext,
3238 unsigned Reg,
3239 BitTestCase &B,
3240 MachineBasicBlock *SwitchBB) {
3241 SDLoc dl = getCurSDLoc();
3242 MVT VT = BB.RegVT;
3243 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3244 SDValue Cmp;
3245 unsigned PopCount = llvm::popcount(B.Mask);
3247 if (PopCount == 1) {
3248 // Testing for a single bit; just compare the shift count with what it
3249 // would need to be to shift a 1 bit in that position.
3250 Cmp = DAG.getSetCC(
3252 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3253 ISD::SETEQ);
3254 } else if (PopCount == BB.Range) {
3255 // There is only one zero bit in the range, test for it directly.
3256 Cmp = DAG.getSetCC(
3258 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3259 } else {
3260 // Make desired shift
3261 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3262 DAG.getConstant(1, dl, VT), ShiftOp);
3263
3264 // Emit bit tests and jumps
3265 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3266 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3267 Cmp = DAG.getSetCC(
3269 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3270 }
3271
3272 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3273 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3274 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3275 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3276 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3277 // one as they are relative probabilities (and thus work more like weights),
3278 // and hence we need to normalize them to let the sum of them become one.
3279 SwitchBB->normalizeSuccProbs();
3280
3281 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3282 MVT::Other, getControlRoot(),
3283 Cmp, DAG.getBasicBlock(B.TargetBB));
3284
3285 // Avoid emitting unnecessary branches to the next block.
3286 if (NextMBB != NextBlock(SwitchBB))
3287 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3288 DAG.getBasicBlock(NextMBB));
3289
3290 DAG.setRoot(BrAnd);
3291}
3292
3293void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3294 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3295
3296 // Retrieve successors. Look through artificial IR level blocks like
3297 // catchswitch for successors.
3298 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
3299 const BasicBlock *EHPadBB = I.getSuccessor(1);
3300 MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
3301
3302 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3303 // have to do anything here to lower funclet bundles.
3304 assert(!I.hasOperandBundlesOtherThan(
3305 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3306 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3307 LLVMContext::OB_cfguardtarget,
3308 LLVMContext::OB_clang_arc_attachedcall}) &&
3309 "Cannot lower invokes with arbitrary operand bundles yet!");
3310
3311 const Value *Callee(I.getCalledOperand());
3312 const Function *Fn = dyn_cast<Function>(Callee);
3313 if (isa<InlineAsm>(Callee))
3314 visitInlineAsm(I, EHPadBB);
3315 else if (Fn && Fn->isIntrinsic()) {
3316 switch (Fn->getIntrinsicID()) {
3317 default:
3318 llvm_unreachable("Cannot invoke this intrinsic");
3319 case Intrinsic::donothing:
3320 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3321 case Intrinsic::seh_try_begin:
3322 case Intrinsic::seh_scope_begin:
3323 case Intrinsic::seh_try_end:
3324 case Intrinsic::seh_scope_end:
3325 if (EHPadMBB)
3326 // a block referenced by EH table
3327 // so dtor-funclet not removed by opts
3328 EHPadMBB->setMachineBlockAddressTaken();
3329 break;
3330 case Intrinsic::experimental_patchpoint_void:
3331 case Intrinsic::experimental_patchpoint_i64:
3332 visitPatchpoint(I, EHPadBB);
3333 break;
3334 case Intrinsic::experimental_gc_statepoint:
3335 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3336 break;
3337 case Intrinsic::wasm_rethrow: {
3338 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3339 // special because it can be invoked, so we manually lower it to a DAG
3340 // node here.
3342 Ops.push_back(getRoot()); // inchain
3344 Ops.push_back(
3345 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3347 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3349 break;
3350 }
3351 }
3352 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
3353 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3354 // Eventually we will support lowering the @llvm.experimental.deoptimize
3355 // intrinsic, and right now there are no plans to support other intrinsics
3356 // with deopt state.
3357 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3358 } else {
3359 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3360 }
3361
3362 // If the value of the invoke is used outside of its defining block, make it
3363 // available as a virtual register.
3364 // We already took care of the exported value for the statepoint instruction
3365 // during call to the LowerStatepoint.
3366 if (!isa<GCStatepointInst>(I)) {
3368 }
3369
3372 BranchProbability EHPadBBProb =
3373 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3375 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3376
3377 // Update successor info.
3378 addSuccessorWithProb(InvokeMBB, Return);
3379 for (auto &UnwindDest : UnwindDests) {
3380 UnwindDest.first->setIsEHPad();
3381 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3382 }
3383 InvokeMBB->normalizeSuccProbs();
3384
3385 // Drop into normal successor.
3387 DAG.getBasicBlock(Return)));
3388}
3389
3390void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3391 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3392
3393 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3394 // have to do anything here to lower funclet bundles.
3395 assert(!I.hasOperandBundlesOtherThan(
3396 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3397 "Cannot lower callbrs with arbitrary operand bundles yet!");
3398
3399 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3400 visitInlineAsm(I);
3402
3403 // Retrieve successors.
3405 Dests.insert(I.getDefaultDest());
3406 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
3407
3408 // Update successor info.
3409 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3410 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3411 BasicBlock *Dest = I.getIndirectDest(i);
3413 Target->setIsInlineAsmBrIndirectTarget();
3414 Target->setMachineBlockAddressTaken();
3415 Target->setLabelMustBeEmitted();
3416 // Don't add duplicate machine successors.
3417 if (Dests.insert(Dest).second)
3418 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3419 }
3420 CallBrMBB->normalizeSuccProbs();
3421
3422 // Drop into default successor.
3424 MVT::Other, getControlRoot(),
3425 DAG.getBasicBlock(Return)));
3426}
3427
3428void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3429 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3430}
3431
3432void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3434 "Call to landingpad not in landing pad!");
3435
3436 // If there aren't registers to copy the values into (e.g., during SjLj
3437 // exceptions), then don't bother to create these DAG nodes.
3439 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3440 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3441 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3442 return;
3443
3444 // If landingpad's return type is token type, we don't create DAG nodes
3445 // for its exception pointer and selector value. The extraction of exception
3446 // pointer or selector value from token type landingpads is not currently
3447 // supported.
3448 if (LP.getType()->isTokenTy())
3449 return;
3450
3451 SmallVector<EVT, 2> ValueVTs;
3452 SDLoc dl = getCurSDLoc();
3453 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3454 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3455
3456 // Get the two live-in registers as SDValues. The physregs have already been
3457 // copied into virtual registers.
3458 SDValue Ops[2];
3460 Ops[0] = DAG.getZExtOrTrunc(
3464 dl, ValueVTs[0]);
3465 } else {
3466 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3467 }
3468 Ops[1] = DAG.getZExtOrTrunc(
3472 dl, ValueVTs[1]);
3473
3474 // Merge into one.
3476 DAG.getVTList(ValueVTs), Ops);
3477 setValue(&LP, Res);
3478}
3479
3482 // Update JTCases.
3483 for (JumpTableBlock &JTB : SL->JTCases)
3484 if (JTB.first.HeaderBB == First)
3485 JTB.first.HeaderBB = Last;
3486
3487 // Update BitTestCases.
3488 for (BitTestBlock &BTB : SL->BitTestCases)
3489 if (BTB.Parent == First)
3490 BTB.Parent = Last;
3491}
3492
3493void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3494 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3495
3496 // Update machine-CFG edges with unique successors.
3498 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3499 BasicBlock *BB = I.getSuccessor(i);
3500 bool Inserted = Done.insert(BB).second;
3501 if (!Inserted)
3502 continue;
3503
3504 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3505 addSuccessorWithProb(IndirectBrMBB, Succ);
3506 }
3507 IndirectBrMBB->normalizeSuccProbs();
3508
3510 MVT::Other, getControlRoot(),
3511 getValue(I.getAddress())));
3512}
3513
3514void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3516 return;
3517
3518 // We may be able to ignore unreachable behind a noreturn call.
3520 if (const CallInst *Call = dyn_cast_or_null<CallInst>(I.getPrevNode())) {
3521 if (Call->doesNotReturn())
3522 return;
3523 }
3524 }
3525
3526 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3527}
3528
3529void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3531 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3532 Flags.copyFMF(*FPOp);
3533
3534 SDValue Op = getValue(I.getOperand(0));
3535 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3536 Op, Flags);
3537 setValue(&I, UnNodeValue);
3538}
3539
3540void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3542 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3543 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3544 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3545 }
3546 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3547 Flags.setExact(ExactOp->isExact());
3548 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3549 Flags.setDisjoint(DisjointOp->isDisjoint());
3550 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3551 Flags.copyFMF(*FPOp);
3552
3553 SDValue Op1 = getValue(I.getOperand(0));
3554 SDValue Op2 = getValue(I.getOperand(1));
3555 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3556 Op1, Op2, Flags);
3557 setValue(&I, BinNodeValue);
3558}
3559
3560void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3561 SDValue Op1 = getValue(I.getOperand(0));
3562 SDValue Op2 = getValue(I.getOperand(1));
3563
3565 Op1.getValueType(), DAG.getDataLayout());
3566
3567 // Coerce the shift amount to the right type if we can. This exposes the
3568 // truncate or zext to optimization early.
3569 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3571 "Unexpected shift type");
3572 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3573 }
3574
3575 bool nuw = false;
3576 bool nsw = false;
3577 bool exact = false;
3578
3579 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3580
3581 if (const OverflowingBinaryOperator *OFBinOp =
3582 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3583 nuw = OFBinOp->hasNoUnsignedWrap();
3584 nsw = OFBinOp->hasNoSignedWrap();
3585 }
3586 if (const PossiblyExactOperator *ExactOp =
3587 dyn_cast<const PossiblyExactOperator>(&I))
3588 exact = ExactOp->isExact();
3589 }
3591 Flags.setExact(exact);
3592 Flags.setNoSignedWrap(nsw);
3593 Flags.setNoUnsignedWrap(nuw);
3594 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3595 Flags);
3596 setValue(&I, Res);
3597}
3598
3599void SelectionDAGBuilder::visitSDiv(const User &I) {
3600 SDValue Op1 = getValue(I.getOperand(0));
3601 SDValue Op2 = getValue(I.getOperand(1));
3602
3604 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3605 cast<PossiblyExactOperator>(&I)->isExact());
3607 Op2, Flags));
3608}
3609
3610void SelectionDAGBuilder::visitICmp(const User &I) {
3612 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3613 predicate = IC->getPredicate();
3614 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3615 predicate = ICmpInst::Predicate(IC->getPredicate());
3616 SDValue Op1 = getValue(I.getOperand(0));
3617 SDValue Op2 = getValue(I.getOperand(1));
3618 ISD::CondCode Opcode = getICmpCondCode(predicate);
3619
3620 auto &TLI = DAG.getTargetLoweringInfo();
3621 EVT MemVT =
3622 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3623
3624 // If a pointer's DAG type is larger than its memory type then the DAG values
3625 // are zero-extended. This breaks signed comparisons so truncate back to the
3626 // underlying type before doing the compare.
3627 if (Op1.getValueType() != MemVT) {
3628 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3629 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3630 }
3631
3633 I.getType());
3634 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3635}
3636
3637void SelectionDAGBuilder::visitFCmp(const User &I) {
3639 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3640 predicate = FC->getPredicate();
3641 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3642 predicate = FCmpInst::Predicate(FC->getPredicate());
3643 SDValue Op1 = getValue(I.getOperand(0));
3644 SDValue Op2 = getValue(I.getOperand(1));
3645
3646 ISD::CondCode Condition = getFCmpCondCode(predicate);
3647 auto *FPMO = cast<FPMathOperator>(&I);
3648 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3649 Condition = getFCmpCodeWithoutNaN(Condition);
3650
3652 Flags.copyFMF(*FPMO);
3653 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3654
3656 I.getType());
3657 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3658}
3659
3660// Check if the condition of the select has one use or two users that are both
3661// selects with the same condition.
3662static bool hasOnlySelectUsers(const Value *Cond) {
3663 return llvm::all_of(Cond->users(), [](const Value *V) {
3664 return isa<SelectInst>(V);
3665 });
3666}
3667
3668void SelectionDAGBuilder::visitSelect(const User &I) {
3669 SmallVector<EVT, 4> ValueVTs;
3671 ValueVTs);
3672 unsigned NumValues = ValueVTs.size();
3673 if (NumValues == 0) return;
3674
3675 SmallVector<SDValue, 4> Values(NumValues);
3676 SDValue Cond = getValue(I.getOperand(0));
3677 SDValue LHSVal = getValue(I.getOperand(1));
3678 SDValue RHSVal = getValue(I.getOperand(2));
3679 SmallVector<SDValue, 1> BaseOps(1, Cond);
3680 ISD::NodeType OpCode =
3681 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3682
3683 bool IsUnaryAbs = false;
3684 bool Negate = false;
3685
3687 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3688 Flags.copyFMF(*FPOp);
3689
3690 Flags.setUnpredictable(
3691 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3692
3693 // Min/max matching is only viable if all output VTs are the same.
3694 if (all_equal(ValueVTs)) {
3695 EVT VT = ValueVTs[0];
3696 LLVMContext &Ctx = *DAG.getContext();
3697 auto &TLI = DAG.getTargetLoweringInfo();
3698
3699 // We care about the legality of the operation after it has been type
3700 // legalized.
3701 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3702 VT = TLI.getTypeToTransformTo(Ctx, VT);
3703
3704 // If the vselect is legal, assume we want to leave this as a vector setcc +
3705 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3706 // min/max is legal on the scalar type.
3707 bool UseScalarMinMax = VT.isVector() &&
3709
3710 // ValueTracking's select pattern matching does not account for -0.0,
3711 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3712 // -0.0 is less than +0.0.
3713 Value *LHS, *RHS;
3714 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3716 switch (SPR.Flavor) {
3717 case SPF_UMAX: Opc = ISD::UMAX; break;
3718 case SPF_UMIN: Opc = ISD::UMIN; break;
3719 case SPF_SMAX: Opc = ISD::SMAX; break;
3720 case SPF_SMIN: Opc = ISD::SMIN; break;
3721 case SPF_FMINNUM:
3722 switch (SPR.NaNBehavior) {
3723 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3724 case SPNB_RETURNS_NAN: break;
3725 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3726 case SPNB_RETURNS_ANY:
3728 (UseScalarMinMax &&
3730 Opc = ISD::FMINNUM;
3731 break;
3732 }
3733 break;
3734 case SPF_FMAXNUM:
3735 switch (SPR.NaNBehavior) {
3736 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3737 case SPNB_RETURNS_NAN: break;
3738 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3739 case SPNB_RETURNS_ANY:
3741 (UseScalarMinMax &&
3743 Opc = ISD::FMAXNUM;
3744 break;
3745 }
3746 break;
3747 case SPF_NABS:
3748 Negate = true;
3749 [[fallthrough]];
3750 case SPF_ABS:
3751 IsUnaryAbs = true;
3752 Opc = ISD::ABS;
3753 break;
3754 default: break;
3755 }
3756
3757 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3758 (TLI.isOperationLegalOrCustomOrPromote(Opc, VT) ||
3759 (UseScalarMinMax &&
3760 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3761 // If the underlying comparison instruction is used by any other
3762 // instruction, the consumed instructions won't be destroyed, so it is
3763 // not profitable to convert to a min/max.
3764 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3765 OpCode = Opc;
3766 LHSVal = getValue(LHS);
3767 RHSVal = getValue(RHS);
3768 BaseOps.clear();
3769 }
3770
3771 if (IsUnaryAbs) {
3772 OpCode = Opc;
3773 LHSVal = getValue(LHS);
3774 BaseOps.clear();
3775 }
3776 }
3777
3778 if (IsUnaryAbs) {
3779 for (unsigned i = 0; i != NumValues; ++i) {
3780 SDLoc dl = getCurSDLoc();
3781 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3782 Values[i] =
3783 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3784 if (Negate)
3785 Values[i] = DAG.getNegative(Values[i], dl, VT);
3786 }
3787 } else {
3788 for (unsigned i = 0; i != NumValues; ++i) {
3789 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3790 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3791 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3792 Values[i] = DAG.getNode(
3793 OpCode, getCurSDLoc(),
3794 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3795 }
3796 }
3797
3799 DAG.getVTList(ValueVTs), Values));
3800}
3801
3802void SelectionDAGBuilder::visitTrunc(const User &I) {
3803 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3804 SDValue N = getValue(I.getOperand(0));
3806 I.getType());
3808}
3809
3810void SelectionDAGBuilder::visitZExt(const User &I) {
3811 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3812 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3813 SDValue N = getValue(I.getOperand(0));
3814 auto &TLI = DAG.getTargetLoweringInfo();
3815 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3816
3818 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3819 Flags.setNonNeg(PNI->hasNonNeg());
3820
3821 // Eagerly use nonneg information to canonicalize towards sign_extend if
3822 // that is the target's preference.
3823 // TODO: Let the target do this later.
3824 if (Flags.hasNonNeg() &&
3825 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
3827 return;
3828 }
3829
3830 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
3831}
3832
3833void SelectionDAGBuilder::visitSExt(const User &I) {
3834 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3835 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3836 SDValue N = getValue(I.getOperand(0));
3838 I.getType());
3840}
3841
3842void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3843 // FPTrunc is never a no-op cast, no need to check
3844 SDValue N = getValue(I.getOperand(0));
3845 SDLoc dl = getCurSDLoc();
3847 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3848 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3850 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3851}
3852
3853void SelectionDAGBuilder::visitFPExt(const User &I) {
3854 // FPExt is never a no-op cast, no need to check
3855 SDValue N = getValue(I.getOperand(0));
3857 I.getType());
3859}
3860
3861void SelectionDAGBuilder::visitFPToUI(const User &I) {
3862 // FPToUI is never a no-op cast, no need to check
3863 SDValue N = getValue(I.getOperand(0));
3865 I.getType());
3867}
3868
3869void SelectionDAGBuilder::visitFPToSI(const User &I) {
3870 // FPToSI is never a no-op cast, no need to check
3871 SDValue N = getValue(I.getOperand(0));
3873 I.getType());
3875}
3876
3877void SelectionDAGBuilder::visitUIToFP(const User &I) {
3878 // UIToFP is never a no-op cast, no need to check
3879 SDValue N = getValue(I.getOperand(0));
3881 I.getType());
3883}
3884
3885void SelectionDAGBuilder::visitSIToFP(const User &I) {
3886 // SIToFP is never a no-op cast, no need to check
3887 SDValue N = getValue(I.getOperand(0));
3889 I.getType());
3891}
3892
3893void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3894 // What to do depends on the size of the integer and the size of the pointer.
3895 // We can either truncate, zero extend, or no-op, accordingly.
3896 SDValue N = getValue(I.getOperand(0));
3897 auto &TLI = DAG.getTargetLoweringInfo();
3899 I.getType());
3900 EVT PtrMemVT =
3901 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3902 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3903 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3904 setValue(&I, N);
3905}
3906
3907void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3908 // What to do depends on the size of the integer and the size of the pointer.
3909 // We can either truncate, zero extend, or no-op, accordingly.
3910 SDValue N = getValue(I.getOperand(0));
3911 auto &TLI = DAG.getTargetLoweringInfo();
3912 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3913 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3914 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3915 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3916 setValue(&I, N);
3917}
3918
3919void SelectionDAGBuilder::visitBitCast(const User &I) {
3920 SDValue N = getValue(I.getOperand(0));
3921 SDLoc dl = getCurSDLoc();
3923 I.getType());
3924
3925 // BitCast assures us that source and destination are the same size so this is
3926 // either a BITCAST or a no-op.
3927 if (DestVT != N.getValueType())
3929 DestVT, N)); // convert types.
3930 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3931 // might fold any kind of constant expression to an integer constant and that
3932 // is not what we are looking for. Only recognize a bitcast of a genuine
3933 // constant integer as an opaque constant.
3934 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3935 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3936 /*isOpaque*/true));
3937 else
3938 setValue(&I, N); // noop cast.
3939}
3940
3941void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3943 const Value *SV = I.getOperand(0);
3944 SDValue N = getValue(SV);
3945 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3946
3947 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3948 unsigned DestAS = I.getType()->getPointerAddressSpace();
3949
3950 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3951 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3952
3953 setValue(&I, N);
3954}
3955
3956void SelectionDAGBuilder::visitInsertElement(const User &I) {
3958 SDValue InVec = getValue(I.getOperand(0));
3959 SDValue InVal = getValue(I.getOperand(1));
3960 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3963 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3964 InVec, InVal, InIdx));
3965}
3966
3967void SelectionDAGBuilder::visitExtractElement(const User &I) {
3969 SDValue InVec = getValue(I.getOperand(0));
3970 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3973 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3974 InVec, InIdx));
3975}
3976
3977void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3978 SDValue Src1 = getValue(I.getOperand(0));
3979 SDValue Src2 = getValue(I.getOperand(1));
3981 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3982 Mask = SVI->getShuffleMask();
3983 else
3984 Mask = cast<ConstantExpr>(I).getShuffleMask();
3985 SDLoc DL = getCurSDLoc();
3987 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3988 EVT SrcVT = Src1.getValueType();
3989
3990 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3991 VT.isScalableVector()) {
3992 // Canonical splat form of first element of first input vector.
3993 SDValue FirstElt =
3996 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3997 return;
3998 }
3999
4000 // For now, we only handle splats for scalable vectors.
4001 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4002 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4003 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4004
4005 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4006 unsigned MaskNumElts = Mask.size();
4007
4008 if (SrcNumElts == MaskNumElts) {
4009 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4010 return;
4011 }
4012
4013 // Normalize the shuffle vector since mask and vector length don't match.
4014 if (SrcNumElts < MaskNumElts) {
4015 // Mask is longer than the source vectors. We can use concatenate vector to
4016 // make the mask and vectors lengths match.
4017
4018 if (MaskNumElts % SrcNumElts == 0) {
4019 // Mask length is a multiple of the source vector length.
4020 // Check if the shuffle is some kind of concatenation of the input
4021 // vectors.
4022 unsigned NumConcat = MaskNumElts / SrcNumElts;
4023 bool IsConcat = true;
4024 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4025 for (unsigned i = 0; i != MaskNumElts; ++i) {
4026 int Idx = Mask[i];
4027 if (Idx < 0)
4028 continue;
4029 // Ensure the indices in each SrcVT sized piece are sequential and that
4030 // the same source is used for the whole piece.
4031 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4032 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4033 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4034 IsConcat = false;
4035 break;
4036 }
4037 // Remember which source this index came from.
4038 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4039 }
4040
4041 // The shuffle is concatenating multiple vectors together. Just emit
4042 // a CONCAT_VECTORS operation.
4043 if (IsConcat) {
4044 SmallVector<SDValue, 8> ConcatOps;
4045 for (auto Src : ConcatSrcs) {
4046 if (Src < 0)
4047 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4048 else if (Src == 0)
4049 ConcatOps.push_back(Src1);
4050 else
4051 ConcatOps.push_back(Src2);
4052 }
4053 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4054 return;
4055 }
4056 }
4057
4058 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4059 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4060 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4061 PaddedMaskNumElts);
4062
4063 // Pad both vectors with undefs to make them the same length as the mask.
4064 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4065
4066 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4067 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4068 MOps1[0] = Src1;
4069 MOps2[0] = Src2;
4070
4071 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4072 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4073
4074 // Readjust mask for new input vector length.
4075 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4076 for (unsigned i = 0; i != MaskNumElts; ++i) {
4077 int Idx = Mask[i];
4078 if (Idx >= (int)SrcNumElts)
4079 Idx -= SrcNumElts - PaddedMaskNumElts;
4080 MappedOps[i] = Idx;
4081 }
4082
4083 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4084
4085 // If the concatenated vector was padded, extract a subvector with the
4086 // correct number of elements.
4087 if (MaskNumElts != PaddedMaskNumElts)
4090
4091 setValue(&I, Result);
4092 return;
4093 }
4094
4095 if (SrcNumElts > MaskNumElts) {
4096 // Analyze the access pattern of the vector to see if we can extract
4097 // two subvectors and do the shuffle.
4098 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
4099 bool CanExtract = true;
4100 for (int Idx : Mask) {
4101 unsigned Input = 0;
4102 if (Idx < 0)
4103 continue;
4104
4105 if (Idx >= (int)SrcNumElts) {
4106 Input = 1;
4107 Idx -= SrcNumElts;
4108 }
4109
4110 // If all the indices come from the same MaskNumElts sized portion of
4111 // the sources we can use extract. Also make sure the extract wouldn't
4112 // extract past the end of the source.
4113 int NewStartIdx = alignDown(Idx, MaskNumElts);
4114 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4115 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4116 CanExtract = false;
4117 // Make sure we always update StartIdx as we use it to track if all
4118 // elements are undef.
4119 StartIdx[Input] = NewStartIdx;
4120 }
4121
4122 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4123 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4124 return;
4125 }
4126 if (CanExtract) {
4127 // Extract appropriate subvector and generate a vector shuffle
4128 for (unsigned Input = 0; Input < 2; ++Input) {
4129 SDValue &Src = Input == 0 ? Src1 : Src2;
4130 if (StartIdx[Input] < 0)
4131 Src = DAG.getUNDEF(VT);
4132 else {
4133 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4134 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4135 }
4136 }
4137
4138 // Calculate new mask.
4139 SmallVector<int, 8> MappedOps(Mask);
4140 for (int &Idx : MappedOps) {
4141 if (Idx >= (int)SrcNumElts)
4142 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4143 else if (Idx >= 0)
4144 Idx -= StartIdx[0];
4145 }
4146
4147 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4148 return;
4149 }
4150 }
4151
4152 // We can't use either concat vectors or extract subvectors so fall back to
4153 // replacing the shuffle with extract and build vector.
4154 // to insert and build vector.
4155 EVT EltVT = VT.getVectorElementType();
4157 for (int Idx : Mask) {
4158 SDValue Res;
4159
4160 if (Idx < 0) {
4161 Res = DAG.getUNDEF(EltVT);
4162 } else {
4163 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4164 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4165
4166 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4168 }
4169
4170 Ops.push_back(Res);
4171 }
4172
4173 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4174}
4175
4176void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4177 ArrayRef<unsigned> Indices = I.getIndices();
4178 const Value *Op0 = I.getOperand(0);
4179 const Value *Op1 = I.getOperand(1);
4180 Type *AggTy = I.getType();
4181 Type *ValTy = Op1->getType();
4182 bool IntoUndef = isa<UndefValue>(Op0);
4183 bool FromUndef = isa<UndefValue>(Op1);
4184
4185 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4186
4188 SmallVector<EVT, 4> AggValueVTs;
4189 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4190 SmallVector<EVT, 4> ValValueVTs;
4191 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4192
4193 unsigned NumAggValues = AggValueVTs.size();
4194 unsigned NumValValues = ValValueVTs.size();
4195 SmallVector<SDValue, 4> Values(NumAggValues);
4196
4197 // Ignore an insertvalue that produces an empty object
4198 if (!NumAggValues) {
4199 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4200 return;
4201 }
4202
4203 SDValue Agg = getValue(Op0);
4204 unsigned i = 0;
4205 // Copy the beginning value(s) from the original aggregate.
4206 for (; i != LinearIndex; ++i)
4207 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4208 SDValue(Agg.getNode(), Agg.getResNo() + i);
4209 // Copy values from the inserted value(s).
4210 if (NumValValues) {
4211 SDValue Val = getValue(Op1);
4212 for (; i != LinearIndex + NumValValues; ++i)
4213 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4214 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4215 }
4216 // Copy remaining value(s) from the original aggregate.
4217 for (; i != NumAggValues; ++i)
4218 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4219 SDValue(Agg.getNode(), Agg.getResNo() + i);
4220
4222 DAG.getVTList(AggValueVTs), Values));
4223}
4224
4225void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4226 ArrayRef<unsigned> Indices = I.getIndices();
4227 const Value *Op0 = I.getOperand(0);
4228 Type *AggTy = Op0->getType();
4229 Type *ValTy = I.getType();
4230 bool OutOfUndef = isa<UndefValue>(Op0);
4231
4232 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4233
4235 SmallVector<EVT, 4> ValValueVTs;
4236 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4237
4238 unsigned NumValValues = ValValueVTs.size();
4239
4240 // Ignore a extractvalue that produces an empty object
4241 if (!NumValValues) {
4242 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4243 return;
4244 }
4245
4246 SmallVector<SDValue, 4> Values(NumValValues);
4247
4248 SDValue Agg = getValue(Op0);
4249 // Copy out the selected value(s).
4250 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4251 Values[i - LinearIndex] =
4252 OutOfUndef ?
4253 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4254 SDValue(Agg.getNode(), Agg.getResNo() + i);
4255
4257 DAG.getVTList(ValValueVTs), Values));
4258}
4259
4260void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4261 Value *Op0 = I.getOperand(0);
4262 // Note that the pointer operand may be a vector of pointers. Take the scalar
4263 // element which holds a pointer.
4264 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4265 SDValue N = getValue(Op0);
4266 SDLoc dl = getCurSDLoc();
4267 auto &TLI = DAG.getTargetLoweringInfo();
4268
4269 // Normalize Vector GEP - all scalar operands should be converted to the
4270 // splat vector.
4271 bool IsVectorGEP = I.getType()->isVectorTy();
4272 ElementCount VectorElementCount =
4273 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4275
4276 if (IsVectorGEP && !N.getValueType().isVector()) {
4278 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
4279 N = DAG.getSplat(VT, dl, N);
4280 }
4281
4283 GTI != E; ++GTI) {
4284 const Value *Idx = GTI.getOperand();
4285 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4286 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4287 if (Field) {
4288 // N = N + Offset
4291
4292 // In an inbounds GEP with an offset that is nonnegative even when
4293 // interpreted as signed, assume there is no unsigned overflow.
4295 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
4296 Flags.setNoUnsignedWrap(true);
4297
4298 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
4299 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
4300 }
4301 } else {
4302 // IdxSize is the width of the arithmetic according to IR semantics.
4303 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4304 // (and fix up the result later).
4305 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4306 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4307 TypeSize ElementSize =
4308 GTI.getSequentialElementStride(DAG.getDataLayout());
4309 // We intentionally mask away the high bits here; ElementSize may not
4310 // fit in IdxTy.
4311 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
4312 bool ElementScalable = ElementSize.isScalable();
4313
4314 // If this is a scalar constant or a splat vector of constants,
4315 // handle it quickly.
4316 const auto *C = dyn_cast<Constant>(Idx);
4317 if (C && isa<VectorType>(C->getType()))
4318 C = C->getSplatValue();
4319
4320 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4321 if (CI && CI->isZero())
4322 continue;
4323 if (CI && !ElementScalable) {
4324 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4326 SDValue OffsVal;
4327 if (IsVectorGEP)
4328 OffsVal = DAG.getConstant(
4329 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4330 else
4331 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4332
4333 // In an inbounds GEP with an offset that is nonnegative even when
4334 // interpreted as signed, assume there is no unsigned overflow.
4336 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
4337 Flags.setNoUnsignedWrap(true);
4338
4339 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4340
4341 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
4342 continue;
4343 }
4344
4345 // N = N + Idx * ElementMul;
4346 SDValue IdxN = getValue(Idx);
4347
4348 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4350 VectorElementCount);
4351 IdxN = DAG.getSplat(VT, dl, IdxN);
4352 }
4353
4354 // If the index is smaller or larger than intptr_t, truncate or extend
4355 // it.
4356 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4357
4358 if (ElementScalable) {
4359 EVT VScaleTy = N.getValueType().getScalarType();
4360 SDValue VScale = DAG.getNode(
4361 ISD::VSCALE, dl, VScaleTy,
4362 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4363 if (IsVectorGEP)
4364 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4365 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
4366 } else {
4367 // If this is a multiply by a power of two, turn it into a shl
4368 // immediately. This is a very common case.
4369 if (ElementMul != 1) {
4370 if (ElementMul.isPowerOf2()) {
4371 unsigned Amt = ElementMul.logBase2();
4372 IdxN = DAG.getNode(ISD::SHL, dl,
4373 N.getValueType(), IdxN,
4374 DAG.getConstant(Amt, dl, IdxN.getValueType()));
4375 } else {
4376 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4377 IdxN.getValueType());
4378 IdxN = DAG.getNode(ISD::MUL, dl,
4379 N.getValueType(), IdxN, Scale);
4380 }
4381 }
4382 }
4383
4384 N = DAG.getNode(ISD::ADD, dl,
4385 N.getValueType(), N, IdxN);
4386 }
4387 }
4388
4389 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4390 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4391 if (IsVectorGEP) {
4392 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4393 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4394 }
4395
4396 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4397 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4398
4399 setValue(&I, N);
4400}
4401
4402void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4403 // If this is a fixed sized alloca in the entry block of the function,
4404 // allocate it statically on the stack.
4405 if (FuncInfo.StaticAllocaMap.count(&I))
4406 return; // getValue will auto-populate this.
4407
4408 SDLoc dl = getCurSDLoc();
4409 Type *Ty = I.getAllocatedType();
4411 auto &DL = DAG.getDataLayout();
4412 TypeSize TySize = DL.getTypeAllocSize(Ty);
4413 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4414
4415 SDValue AllocSize = getValue(I.getArraySize());
4416
4417 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4418 if (AllocSize.getValueType() != IntPtr)
4419 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4420
4421 if (TySize.isScalable())
4422 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4423 DAG.getVScale(dl, IntPtr,
4424 APInt(IntPtr.getScalarSizeInBits(),
4425 TySize.getKnownMinValue())));
4426 else {
4427 SDValue TySizeValue =
4429 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4430 DAG.getZExtOrTrunc(TySizeValue, dl, IntPtr));
4431 }
4432
4433 // Handle alignment. If the requested alignment is less than or equal to
4434 // the stack alignment, ignore it. If the size is greater than or equal to
4435 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4437 if (*Alignment <= StackAlign)
4438 Alignment = std::nullopt;
4439
4440 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4441 // Round the size of the allocation up to the stack alignment size
4442 // by add SA-1 to the size. This doesn't overflow because we're computing
4443 // an address inside an alloca.
4445 Flags.setNoUnsignedWrap(true);
4446 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4447 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4448
4449 // Mask out the low bits for alignment purposes.
4450 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4451 DAG.getConstant(~StackAlignMask, dl, IntPtr));
4452
4453 SDValue Ops[] = {
4454 getRoot(), AllocSize,
4455 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4456 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4457 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4458 setValue(&I, DSA);
4459 DAG.setRoot(DSA.getValue(1));
4460
4462}
4463
4464static const MDNode *getRangeMetadata(const Instruction &I) {
4465 // If !noundef is not present, then !range violation results in a poison
4466 // value rather than immediate undefined behavior. In theory, transferring
4467 // these annotations to SDAG is fine, but in practice there are key SDAG
4468 // transforms that are known not to be poison-safe, such as folding logical
4469 // and/or to bitwise and/or. For now, only transfer !range if !noundef is
4470 // also present.
4471 if (!I.hasMetadata(LLVMContext::MD_noundef))
4472 return nullptr;
4473 return I.getMetadata(LLVMContext::MD_range);
4474}
4475
4476void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4477 if (I.isAtomic())
4478 return visitAtomicLoad(I);
4479
4481 const Value *SV = I.getOperand(0);
4482 if (TLI.supportSwiftError()) {
4483 // Swifterror values can come from either a function parameter with
4484 // swifterror attribute or an alloca with swifterror attribute.
4485 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4486 if (Arg->hasSwiftErrorAttr())
4487 return visitLoadFromSwiftError(I);
4488 }
4489
4490 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4491 if (Alloca->isSwiftError())
4492 return visitLoadFromSwiftError(I);
4493 }
4494 }
4495
4496 SDValue Ptr = getValue(SV);
4497
4498 Type *Ty = I.getType();
4499 SmallVector<EVT, 4> ValueVTs, MemVTs;
4501 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4502 unsigned NumValues = ValueVTs.size();
4503 if (NumValues == 0)
4504 return;
4505
4506 Align Alignment = I.getAlign();
4507 AAMDNodes AAInfo = I.getAAMetadata();
4508 const MDNode *Ranges = getRangeMetadata(I);
4509 bool isVolatile = I.isVolatile();
4510 MachineMemOperand::Flags MMOFlags =
4512
4513 SDValue Root;
4514 bool ConstantMemory = false;
4515 if (isVolatile)
4516 // Serialize volatile loads with other side effects.
4517 Root = getRoot();
4518 else if (NumValues > MaxParallelChains)
4519 Root = getMemoryRoot();
4520 else if (AA &&
4522 SV,
4524 AAInfo))) {
4525 // Do not serialize (non-volatile) loads of constant memory with anything.
4526 Root = DAG.getEntryNode();
4527 ConstantMemory = true;
4529 } else {
4530 // Do not serialize non-volatile loads against each other.
4531 Root = DAG.getRoot();
4532 }
4533
4534 SDLoc dl = getCurSDLoc();
4535
4536 if (isVolatile)
4537 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4538
4539 SmallVector<SDValue, 4> Values(NumValues);
4540 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4541
4542 unsigned ChainI = 0;
4543 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4544 // Serializing loads here may result in excessive register pressure, and
4545 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4546 // could recover a bit by hoisting nodes upward in the chain by recognizing
4547 // they are side-effect free or do not alias. The optimizer should really
4548 // avoid this case by converting large object/array copies to llvm.memcpy
4549 // (MaxParallelChains should always remain as failsafe).
4550 if (ChainI == MaxParallelChains) {
4551 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4552 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4553 ArrayRef(Chains.data(), ChainI));
4554 Root = Chain;
4555 ChainI = 0;
4556 }
4557
4558 // TODO: MachinePointerInfo only supports a fixed length offset.
4559 MachinePointerInfo PtrInfo =
4560 !Offsets[i].isScalable() || Offsets[i].isZero()
4561 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4563
4564 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4565 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
4566 MMOFlags, AAInfo, Ranges);
4567 Chains[ChainI] = L.getValue(1);
4568
4569 if (MemVTs[i] != ValueVTs[i])
4570 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4571
4572 Values[i] = L;
4573 }
4574
4575 if (!ConstantMemory) {
4576 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4577 ArrayRef(Chains.data(), ChainI));
4578 if (isVolatile)
4579 DAG.setRoot(Chain);
4580 else
4581 PendingLoads.push_back(Chain);
4582 }
4583
4585 DAG.getVTList(ValueVTs), Values));
4586}
4587
4588void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4590 "call visitStoreToSwiftError when backend supports swifterror");
4591
4592 SmallVector<EVT, 4> ValueVTs;
4594 const Value *SrcV = I.getOperand(0);
4596 SrcV->getType(), ValueVTs, &Offsets, 0);
4597 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4598 "expect a single EVT for swifterror");
4599
4600 SDValue Src = getValue(SrcV);
4601 // Create a virtual register, then update the virtual register.
4602 Register VReg =
4603 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4604 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4605 // Chain can be getRoot or getControlRoot.
4606 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4607 SDValue(Src.getNode(), Src.getResNo()));
4608 DAG.setRoot(CopyNode);
4609}
4610
4611void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4613 "call visitLoadFromSwiftError when backend supports swifterror");
4614
4615 assert(!I.isVolatile() &&
4616 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4617 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4618 "Support volatile, non temporal, invariant for load_from_swift_error");
4619
4620 const Value *SV = I.getOperand(0);
4621 Type *Ty = I.getType();
4622 assert(
4623 (!AA ||
4626 I.getAAMetadata()))) &&
4627 "load_from_swift_error should not be constant memory");
4628
4629 SmallVector<EVT, 4> ValueVTs;
4632 ValueVTs, &Offsets, 0);
4633 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4634 "expect a single EVT for swifterror");
4635
4636 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4638 getRoot(), getCurSDLoc(),
4639 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4640
4641 setValue(&I, L);
4642}
4643
4644void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4645 if (I.isAtomic())
4646 return visitAtomicStore(I);
4647
4648 const Value *SrcV = I.getOperand(0);
4649 const Value *PtrV = I.getOperand(1);
4650
4652 if (TLI.supportSwiftError()) {
4653 // Swifterror values can come from either a function parameter with
4654 // swifterror attribute or an alloca with swifterror attribute.
4655 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4656 if (Arg->hasSwiftErrorAttr())
4657 return visitStoreToSwiftError(I);
4658 }
4659
4660 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4661 if (Alloca->isSwiftError())
4662 return visitStoreToSwiftError(I);
4663 }
4664 }
4665
4666 SmallVector<EVT, 4> ValueVTs, MemVTs;
4669 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4670 unsigned NumValues = ValueVTs.size();
4671 if (NumValues == 0)
4672 return;
4673
4674 // Get the lowered operands. Note that we do this after
4675 // checking if NumResults is zero, because with zero results
4676 // the operands won't have values in the map.
4677 SDValue Src = getValue(SrcV);
4678 SDValue Ptr = getValue(PtrV);
4679
4680 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4681 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4682 SDLoc dl = getCurSDLoc();
4683 Align Alignment = I.getAlign();
4684 AAMDNodes AAInfo = I.getAAMetadata();
4685
4686 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4687
4688 unsigned ChainI = 0;
4689 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4690 // See visitLoad comments.
4691 if (ChainI == MaxParallelChains) {
4692 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4693 ArrayRef(Chains.data(), ChainI));
4694 Root = Chain;
4695 ChainI = 0;
4696 }
4697
4698 // TODO: MachinePointerInfo only supports a fixed length offset.
4699 MachinePointerInfo PtrInfo =
4700 !Offsets[i].isScalable() || Offsets[i].isZero()
4701 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4703
4704 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4705 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4706 if (MemVTs[i] != ValueVTs[i])
4707 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4708 SDValue St =
4709 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4710 Chains[ChainI] = St;
4711 }
4712
4713 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4714 ArrayRef(Chains.data(), ChainI));
4715 setValue(&I, StoreNode);
4716 DAG.setRoot(StoreNode);
4717}
4718
4719void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4720 bool IsCompressing) {
4721 SDLoc sdl = getCurSDLoc();
4722
4723 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4724 Align &Alignment) {
4725 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4726 Src0 = I.getArgOperand(0);
4727 Ptr = I.getArgOperand(1);
4728 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getAlignValue();
4729 Mask = I.getArgOperand(3);
4730 };
4731 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4732 Align &Alignment) {
4733 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4734 Src0 = I.getArgOperand(0);
4735 Ptr = I.getArgOperand(1);
4736 Mask = I.getArgOperand(2);
4737 Alignment = I.getParamAlign(1).valueOrOne();
4738 };
4739
4740 Value *PtrOperand, *MaskOperand, *Src0Operand;
4741 Align Alignment;
4742 if (IsCompressing)
4743 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4744 else
4745 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4746
4747 SDValue Ptr = getValue(PtrOperand);
4748 SDValue Src0 = getValue(Src0Operand);
4749 SDValue Mask = getValue(MaskOperand);
4750 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4751
4752 EVT VT = Src0.getValueType();
4753
4756 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
4757 SDValue StoreNode =
4758 DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
4759 ISD::UNINDEXED, false /* Truncating */, IsCompressing);
4760 DAG.setRoot(StoreNode);
4761 setValue(&I, StoreNode);
4762}
4763
4764// Get a uniform base for the Gather/Scatter intrinsic.
4765// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4766// We try to represent it as a base pointer + vector of indices.
4767// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4768// The first operand of the GEP may be a single pointer or a vector of pointers
4769// Example:
4770// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4771// or
4772// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4773// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4774//
4775// When the first GEP operand is a single pointer - it is the uniform base we
4776// are looking for. If first operand of the GEP is a splat vector - we
4777// extract the splat value and use it as a uniform base.
4778// In all other cases the function returns 'false'.
4780 ISD::MemIndexType &IndexType, SDValue &Scale,
4781 SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4782 uint64_t ElemSize) {
4783 SelectionDAG& DAG = SDB->DAG;
4784 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4785 const DataLayout &DL = DAG.getDataLayout();
4786
4787 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4788
4789 // Handle splat constant pointer.
4790 if (auto *C = dyn_cast<Constant>(Ptr)) {
4791 C = C->getSplatValue();
4792 if (!C)
4793 return false;
4794
4795 Base = SDB->getValue(C);
4796
4797 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4798 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4799 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4800 IndexType = ISD::SIGNED_SCALED;
4801 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4802 return true;
4803 }
4804
4805 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4806 if (!GEP || GEP->getParent() != CurBB)
4807 return false;
4808
4809 if (GEP->getNumOperands() != 2)
4810 return false;
4811
4812 const Value *BasePtr = GEP->getPointerOperand();
4813 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4814
4815 // Make sure the base is scalar and the index is a vector.
4816 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4817 return false;
4818
4819 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4820 if (ScaleVal.isScalable())
4821 return false;
4822
4823 // Target may not support the required addressing mode.
4824 if (ScaleVal != 1 &&
4825 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
4826 return false;
4827
4828 Base = SDB->getValue(BasePtr);
4829 Index = SDB->getValue(IndexVal);
4830 IndexType = ISD::SIGNED_SCALED;
4831
4832 Scale =
4833 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4834 return true;
4835}
4836
4837void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4838 SDLoc sdl = getCurSDLoc();
4839
4840 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4841 const Value *Ptr = I.getArgOperand(1);
4842 SDValue Src0 = getValue(I.getArgOperand(0));
4843 SDValue Mask = getValue(I.getArgOperand(3));
4844 EVT VT = Src0.getValueType();
4845 Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4846 ->getMaybeAlignValue()
4847 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4849
4850 SDValue Base;
4851 SDValue Index;
4852 ISD::MemIndexType IndexType;
4853 SDValue Scale;
4854 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4855 I.getParent(), VT.getScalarStoreSize());
4856
4857 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4860 // TODO: Make MachineMemOperands aware of scalable
4861 // vectors.
4862 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
4863 if (!UniformBase) {
4865 Index = getValue(Ptr);
4866 IndexType = ISD::SIGNED_SCALED;
4867 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4868 }
4869
4870 EVT IdxVT = Index.getValueType();
4871 EVT EltTy = IdxVT.getVectorElementType();
4872 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4873 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4874 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4875 }
4876
4877 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4878 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4879 Ops, MMO, IndexType, false);
4880 DAG.setRoot(Scatter);
4881 setValue(&I, Scatter);
4882}
4883
4884void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4885 SDLoc sdl = getCurSDLoc();
4886
4887 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4888 Align &Alignment) {
4889 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4890 Ptr = I.getArgOperand(0);
4891 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getAlignValue();
4892 Mask = I.getArgOperand(2);
4893 Src0 = I.getArgOperand(3);
4894 };
4895 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4896 Align &Alignment) {
4897 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4898 Ptr = I.getArgOperand(0);
4899 Alignment = I.getParamAlign(0).valueOrOne();
4900 Mask = I.getArgOperand(1);
4901 Src0 = I.getArgOperand(2);
4902 };
4903
4904 Value *PtrOperand, *MaskOperand, *Src0Operand;
4905 Align Alignment;
4906 if (IsExpanding)
4907 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4908 else
4909 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4910
4911 SDValue Ptr = getValue(PtrOperand);
4912 SDValue Src0 = getValue(Src0Operand);
4913 SDValue Mask = getValue(MaskOperand);
4914 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4915
4916 EVT VT = Src0.getValueType();
4917 AAMDNodes AAInfo = I.getAAMetadata();
4918 const MDNode *Ranges = getRangeMetadata(I);
4919
4920 // Do not serialize masked loads of constant memory with anything.
4921 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
4922 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
4923
4924 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4925
4928 MemoryLocation::UnknownSize, Alignment, AAInfo, Ranges);
4929
4930 SDValue Load =
4931 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
4932 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
4933 if (AddToChain)
4934 PendingLoads.push_back(Load.getValue(1));
4935 setValue(&I, Load);
4936}
4937
4938void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4939 SDLoc sdl = getCurSDLoc();
4940
4941 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4942 const Value *Ptr = I.getArgOperand(0);
4943 SDValue Src0 = getValue(I.getArgOperand(3));
4944 SDValue Mask = getValue(I.getArgOperand(2));
4945
4947 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4948 Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
4949 ->getMaybeAlignValue()
4950 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4951
4952 const MDNode *Ranges = getRangeMetadata(I);
4953
4954 SDValue Root = DAG.getRoot();
4955 SDValue Base;
4956 SDValue Index;
4957 ISD::MemIndexType IndexType;
4958 SDValue Scale;
4959 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4960 I.getParent(), VT.getScalarStoreSize());
4961 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4964 // TODO: Make MachineMemOperands aware of scalable
4965 // vectors.
4966 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
4967
4968 if (!UniformBase) {
4970 Index = getValue(Ptr);
4971 IndexType = ISD::SIGNED_SCALED;
4972 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4973 }
4974
4975 EVT IdxVT = Index.getValueType();
4976 EVT EltTy = IdxVT.getVectorElementType();
4977 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4978 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4979 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4980 }
4981
4982 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4983 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4984 Ops, MMO, IndexType, ISD::NON_EXTLOAD);
4985
4986 PendingLoads.push_back(Gather.getValue(1));
4987 setValue(&I, Gather);
4988}
4989
4990void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4991 SDLoc dl = getCurSDLoc();
4992 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4993 AtomicOrdering FailureOrdering = I.getFailureOrdering();
4994 SyncScope::ID SSID = I.getSyncScopeID();
4995
4996 SDValue InChain = getRoot();
4997
4998 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4999 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5000
5003
5006 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5007 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
5008 FailureOrdering);
5009
5011 dl, MemVT, VTs, InChain,
5012 getValue(I.getPointerOperand()),
5013 getValue(I.getCompareOperand()),
5014 getValue(I.getNewValOperand()), MMO);
5015
5016 SDValue OutChain = L.getValue(2);
5017
5018 setValue(&I, L);
5019 DAG.setRoot(OutChain);
5020}
5021
5022void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5023 SDLoc dl = getCurSDLoc();
5025 switch (I.getOperation()) {
5026 default: llvm_unreachable("Unknown atomicrmw operation");
5044 break;
5047 break;
5048 }
5049 AtomicOrdering Ordering = I.getOrdering();
5050 SyncScope::ID SSID = I.getSyncScopeID();
5051
5052 SDValue InChain = getRoot();
5053
5054 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5057
5060 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5061 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);
5062
5063 SDValue L =
5064 DAG.getAtomic(NT, dl, MemVT, InChain,
5065 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5066 MMO);
5067
5068 SDValue OutChain = L.getValue(1);
5069
5070 setValue(&I, L);
5071 DAG.setRoot(OutChain);
5072}
5073
5074void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5075 SDLoc dl = getCurSDLoc();
5077 SDValue Ops[3];
5078 Ops[0] = getRoot();
5079 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5081 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5083 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5084 setValue(&I, N);
5085 DAG.setRoot(N);
5086}
5087
5088void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5089 SDLoc dl = getCurSDLoc();
5090 AtomicOrdering Order = I.getOrdering();
5091 SyncScope::ID SSID = I.getSyncScopeID();
5092
5093 SDValue InChain = getRoot();
5094
5096 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5097 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5098
5099 if (!TLI.supportsUnalignedAtomics() &&
5100 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5101 report_fatal_error("Cannot generate unaligned atomic load");
5102
5104
5106 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5107 I.getAlign(), AAMDNodes(), nullptr, SSID, Order);
5108
5109 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5110
5111 SDValue Ptr = getValue(I.getPointerOperand());
5112 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
5113 Ptr, MMO);
5114
5115 SDValue OutChain = L.getValue(1);
5116 if (MemVT != VT)
5117 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5118
5119 setValue(&I, L);
5120 DAG.setRoot(OutChain);
5121}
5122
5123void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5124 SDLoc dl = getCurSDLoc();
5125
5126 AtomicOrdering Ordering = I.getOrdering();
5127 SyncScope::ID SSID = I.getSyncScopeID();
5128
5129 SDValue InChain = getRoot();
5130
5132 EVT MemVT =
5133 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5134
5135 if (!TLI.supportsUnalignedAtomics() &&
5136 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5137 report_fatal_error("Cannot generate unaligned atomic store");
5138
5140
5143 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5144 I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
5145
5146 SDValue Val = getValue(I.getValueOperand());
5147 if (Val.getValueType() != MemVT)
5148 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5149 SDValue Ptr = getValue(I.getPointerOperand());
5150
5151 SDValue OutChain =
5152 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5153
5154 setValue(&I, OutChain);
5155 DAG.setRoot(OutChain);
5156}
5157
5158/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5159/// node.
5160void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5161 unsigned Intrinsic) {
5162 // Ignore the callsite's attributes. A specific call site may be marked with
5163 // readnone, but the lowering code will expect the chain based on the
5164 // definition.
5165 const Function *F = I.getCalledFunction();
5166 bool HasChain = !F->doesNotAccessMemory();
5167 bool OnlyLoad = HasChain && F->onlyReadsMemory();
5168
5169 // Build the operand list.
5171 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5172 if (OnlyLoad) {
5173 // We don't need to serialize loads against other loads.
5174 Ops.push_back(DAG.getRoot());
5175 } else {
5176 Ops.push_back(getRoot());
5177 }
5178 }
5179
5180 // Info is set by getTgtMemIntrinsic
5183 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
5185 Intrinsic);
5186
5187 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5188 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
5190 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
5192
5193 // Add all operands of the call to the operand list.
5194 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5195 const Value *Arg = I.getArgOperand(i);
5196 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5197 Ops.push_back(getValue(Arg));
5198 continue;
5199 }
5200
5201 // Use TargetConstant instead of a regular constant for immarg.
5202 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5203 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5204 assert(CI->getBitWidth() <= 64 &&
5205 "large intrinsic immediates not handled");
5206 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5207 } else {
5208 Ops.push_back(
5209 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5210 }
5211 }
5212
5213 SmallVector<EVT, 4> ValueVTs;
5214 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5215
5216 if (HasChain)
5217 ValueVTs.push_back(MVT::Other);
5218
5219 SDVTList VTs = DAG.getVTList(ValueVTs);
5220
5221 // Propagate fast-math-flags from IR to node(s).
5223 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5224 Flags.copyFMF(*FPMO);
5225 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5226
5227 // Create the node.
5229
5230 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5231 auto *Token = Bundle->Inputs[0].get();
5232 SDValue ConvControlToken = getValue(Token);
5233 assert(Ops.back().getValueType() != MVT::Glue &&
5234 "Did not expected another glue node here.");
5235 ConvControlToken =
5236 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5237 Ops.push_back(ConvControlToken);
5238 }
5239
5240 // In some cases, custom collection of operands from CallInst I may be needed.
5242 if (IsTgtIntrinsic) {
5243 // This is target intrinsic that touches memory
5244 //
5245 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
5246 // didn't yield anything useful.
5248 if (Info.ptrVal)
5249 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5250 else if (Info.fallbackAddressSpace)
5251 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5252 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
5253 Info.memVT, MPI, Info.align, Info.flags,
5254 Info.size, I.getAAMetadata());
5255 } else if (!HasChain) {
5257 } else if (!I.getType()->isVoidTy()) {
5259 } else {
5261 }
5262
5263 if (HasChain) {
5264 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
5265 if (OnlyLoad)
5266 PendingLoads.push_back(Chain);
5267 else
5268 DAG.setRoot(Chain);
5269 }
5270
5271 if (!I.getType()->isVoidTy()) {
5272 if (!isa<VectorType>(I.getType()))
5273 Result = lowerRangeToAssertZExt(DAG, I, Result);
5274
5275 MaybeAlign Alignment = I.getRetAlign();
5276
5277 // Insert `assertalign` node if there's an alignment.
5278 if (InsertAssertAlign && Alignment) {
5279 Result =
5280 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5281 }
5282
5283 setValue(&I, Result);
5284 }
5285}
5286
5287/// GetSignificand - Get the significand and build it into a floating-point
5288/// number with exponent of 1:
5289///
5290/// Op = (Op & 0x007fffff) | 0x3f800000;
5291///
5292/// where Op is the hexadecimal representation of floating point value.
5294 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5295 DAG.getConstant(0x007fffff, dl, MVT::i32));
5296 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5297 DAG.getConstant(0x3f800000, dl, MVT::i32));
5298 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5299}
5300
5301/// GetExponent - Get the exponent:
5302///
5303/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5304///
5305/// where Op is the hexadecimal representation of floating point value.
5307 const TargetLowering &TLI, const SDLoc &dl) {
5308 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5309 DAG.getConstant(0x7f800000, dl, MVT::i32));
5310 SDValue t1 = DAG.getNode(
5311 ISD::SRL, dl, MVT::i32, t0,
5312 DAG.getConstant(23, dl,
5313 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
5314 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5315 DAG.getConstant(127, dl, MVT::i32));
5316 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5317}
5318
5319/// getF32Constant - Get 32-bit floating point constant.
5321 const SDLoc &dl) {
5322 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5323 MVT::f32);
5324}
5325
5327 SelectionDAG &DAG) {
5328 // TODO: What fast-math-flags should be set on the floating-point nodes?
5329
5330 // IntegerPartOfX = ((int32_t)(t0);
5331 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5332
5333 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5334 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5335 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5336
5337 // IntegerPartOfX <<= 23;
5338 IntegerPartOfX =
5339 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5340 DAG.getConstant(23, dl,
5342 MVT::i32, DAG.getDataLayout())));
5343
5344 SDValue TwoToFractionalPartOfX;
5345 if (LimitFloatPrecision <= 6) {
5346 // For floating-point precision of 6:
5347 //
5348 // TwoToFractionalPartOfX =
5349 // 0.997535578f +
5350 // (0.735607626f + 0.252464424f * x) * x;
5351 //
5352 // error 0.0144103317, which is 6 bits
5353 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5354 getF32Constant(DAG, 0x3e814304, dl));
5355 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5356 getF32Constant(DAG, 0x3f3c50c8, dl));
5357 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5358 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5359 getF32Constant(DAG, 0x3f7f5e7e, dl));
5360 } else if (LimitFloatPrecision <= 12) {
5361 // For floating-point precision of 12:
5362 //
5363 // TwoToFractionalPartOfX =
5364 // 0.999892986f +
5365 // (0.696457318f +
5366 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5367 //
5368 // error 0.000107046256, which is 13 to 14 bits
5369 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5370 getF32Constant(DAG, 0x3da235e3, dl));
5371 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5372 getF32Constant(DAG, 0x3e65b8f3, dl));
5373 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5374 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5375 getF32Constant(DAG, 0x3f324b07, dl));
5376 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5377 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5378 getF32Constant(DAG, 0x3f7ff8fd, dl));
5379 } else { // LimitFloatPrecision <= 18
5380 // For floating-point precision of 18:
5381 //
5382 // TwoToFractionalPartOfX =
5383 // 0.999999982f +
5384 // (0.693148872f +
5385 // (0.240227044f +
5386 // (0.554906021e-1f +
5387 // (0.961591928e-2f +
5388 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5389 // error 2.47208000*10^(-7), which is better than 18 bits
5390 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5391 getF32Constant(DAG, 0x3924b03e, dl));
5392 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5393 getF32Constant(DAG, 0x3ab24b87, dl));
5394 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5395 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5396 getF32Constant(DAG, 0x3c1d8c17, dl));
5397 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5398 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5399 getF32Constant(DAG, 0x3d634a1d, dl));
5400 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5401 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5402 getF32Constant(DAG, 0x3e75fe14, dl));
5403 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5404 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5405 getF32Constant(DAG, 0x3f317234, dl));
5406 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5407 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5408 getF32Constant(DAG, 0x3f800000, dl));
5409 }
5410
5411 // Add the exponent into the result in integer domain.
5412 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5413 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5414 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5415}
5416
5417/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5418/// limited-precision mode.
5420 const TargetLowering &TLI, SDNodeFlags Flags) {
5421 if (Op.getValueType() == MVT::f32 &&
5423
5424 // Put the exponent in the right bit position for later addition to the
5425 // final result:
5426 //
5427 // t0 = Op * log2(e)
5428
5429 // TODO: What fast-math-flags should be set here?
5430 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5431 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5432 return getLimitedPrecisionExp2(t0, dl, DAG);
5433 }
5434
5435 // No special expansion.
5436 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5437}
5438
5439/// expandLog - Lower a log intrinsic. Handles the special sequences for
5440/// limited-precision mode.
5442 const TargetLowering &TLI, SDNodeFlags Flags) {
5443 // TODO: What fast-math-flags should be set on the floating-point nodes?
5444
5445 if (Op.getValueType() == MVT::f32 &&
5447 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5448
5449 // Scale the exponent by log(2).
5450 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5451 SDValue LogOfExponent =
5452 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5453 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5454
5455 // Get the significand and build it into a floating-point number with
5456 // exponent of 1.
5457 SDValue X = GetSignificand(DAG, Op1, dl);
5458
5459 SDValue LogOfMantissa;
5460 if (LimitFloatPrecision <= 6) {
5461 // For floating-point precision of 6:
5462 //
5463 // LogofMantissa =
5464 // -1.1609546f +
5465 // (1.4034025f - 0.23903021f * x) * x;
5466 //
5467 // error 0.0034276066, which is better than 8 bits
5468 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5469 getF32Constant(DAG, 0xbe74c456, dl));
5470 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5471 getF32Constant(DAG, 0x3fb3a2b1, dl));
5472 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5473 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5474 getF32Constant(DAG, 0x3f949a29, dl));
5475 } else if (LimitFloatPrecision <= 12) {
5476 // For floating-point precision of 12:
5477 //
5478 // LogOfMantissa =
5479 // -1.7417939f +
5480 // (2.8212026f +
5481 // (-1.4699568f +
5482 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5483 //
5484 // error 0.000061011436, which is 14 bits
5485 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5486 getF32Constant(DAG, 0xbd67b6d6, dl));
5487 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5488 getF32Constant(DAG, 0x3ee4f4b8, dl));
5489 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5490 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5491 getF32Constant(DAG, 0x3fbc278b, dl));
5492 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5493 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5494 getF32Constant(DAG, 0x40348e95, dl));
5495 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5496 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5497 getF32Constant(DAG, 0x3fdef31a, dl));
5498 } else { // LimitFloatPrecision <= 18
5499 // For floating-point precision of 18:
5500 //
5501 // LogOfMantissa =
5502 // -2.1072184f +
5503 // (4.2372794f +
5504 // (-3.7029485f +
5505 // (2.2781945f +
5506 // (-0.87823314f +
5507 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5508 //
5509 // error 0.0000023660568, which is better than 18 bits
5510 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5511 getF32Constant(DAG, 0xbc91e5ac, dl));
5512 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5513 getF32Constant(DAG, 0x3e4350aa, dl));
5514 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5515 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5516 getF32Constant(DAG, 0x3f60d3e3, dl));
5517 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5518 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5519 getF32Constant(DAG, 0x4011cdf0, dl));
5520 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5521 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5522 getF32Constant(DAG, 0x406cfd1c, dl));
5523 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5524 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5525 getF32Constant(DAG, 0x408797cb, dl));
5526 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5527 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5528 getF32Constant(DAG, 0x4006dcab, dl));
5529 }
5530
5531 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5532 }
5533
5534 // No special expansion.
5535 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5536}
5537
5538/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5539/// limited-precision mode.
5541 const TargetLowering &TLI, SDNodeFlags Flags) {
5542 // TODO: What fast-math-flags should be set on the floating-point nodes?
5543
5544 if (Op.getValueType() == MVT::f32 &&
5546 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5547
5548 // Get the exponent.
5549 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5550
5551 // Get the significand and build it into a floating-point number with
5552 // exponent of 1.
5553 SDValue X = GetSignificand(DAG, Op1, dl);
5554
5555 // Different possible minimax approximations of significand in
5556 // floating-point for various degrees of accuracy over [1,2].
5557 SDValue Log2ofMantissa;
5558 if (LimitFloatPrecision <= 6) {
5559 // For floating-point precision of 6:
5560 //
5561 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5562 //
5563 // error 0.0049451742, which is more than 7 bits
5564 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5565 getF32Constant(DAG, 0xbeb08fe0, dl));
5566 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5567 getF32Constant(DAG, 0x40019463, dl));
5568 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5569 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5570 getF32Constant(DAG, 0x3fd6633d, dl));
5571 } else if (LimitFloatPrecision <= 12) {
5572 // For floating-point precision of 12:
5573 //
5574 // Log2ofMantissa =
5575 // -2.51285454f +
5576 // (4.07009056f +
5577 // (-2.12067489f +
5578 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5579 //
5580 // error 0.0000876136000, which is better than 13 bits
5581 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5582 getF32Constant(DAG, 0xbda7262e, dl));
5583 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5584 getF32Constant(DAG, 0x3f25280b, dl));
5585 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5586 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5587 getF32Constant(DAG, 0x4007b923, dl));
5588 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5589 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5590 getF32Constant(DAG, 0x40823e2f, dl));
5591 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5592 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5593 getF32Constant(DAG, 0x4020d29c, dl));
5594 } else { // LimitFloatPrecision <= 18
5595 // For floating-point precision of 18:
5596 //
5597 // Log2ofMantissa =
5598 // -3.0400495f +
5599 // (6.1129976f +
5600 // (-5.3420409f +
5601 // (3.2865683f +
5602 // (-1.2669343f +
5603 // (0.27515199f -
5604 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5605 //
5606 // error 0.0000018516, which is better than 18 bits
5607 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5608 getF32Constant(DAG, 0xbcd2769e, dl));
5609 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5610 getF32Constant(DAG, 0x3e8ce0b9, dl));
5611 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5612 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5613 getF32Constant(DAG, 0x3fa22ae7, dl));
5614 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5615 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5616 getF32Constant(DAG, 0x40525723, dl));
5617 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5618 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5619 getF32Constant(DAG, 0x40aaf200, dl));
5620 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5621 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5622 getF32Constant(DAG, 0x40c39dad, dl));
5623 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5624 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5625 getF32Constant(DAG, 0x4042902c, dl));
5626 }
5627
5628 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5629 }
5630
5631 // No special expansion.
5632 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5633}
5634
5635/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5636/// limited-precision mode.
5638 const TargetLowering &TLI, SDNodeFlags Flags) {
5639 // TODO: What fast-math-flags should be set on the floating-point nodes?
5640
5641 if (Op.getValueType() == MVT::f32 &&
5643 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5644
5645 // Scale the exponent by log10(2) [0.30102999f].
5646 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5647 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5648 getF32Constant(DAG, 0x3e9a209a, dl));
5649
5650 // Get the significand and build it into a floating-point number with
5651 // exponent of 1.
5652 SDValue X = GetSignificand(DAG, Op1, dl);
5653
5654 SDValue Log10ofMantissa;
5655 if (LimitFloatPrecision <= 6) {
5656 // For floating-point precision of 6:
5657 //
5658 // Log10ofMantissa =
5659 // -0.50419619f +
5660 // (0.60948995f - 0.10380950f * x) * x;
5661 //
5662 // error 0.0014886165, which is 6 bits
5663 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5664 getF32Constant(DAG, 0xbdd49a13, dl));
5665 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5666 getF32Constant(DAG, 0x3f1c0789, dl));
5667 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5668 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5669 getF32Constant(DAG, 0x3f011300, dl));
5670 } else if (LimitFloatPrecision <= 12) {
5671 // For floating-point precision of 12:
5672 //
5673 // Log10ofMantissa =
5674 // -0.64831180f +
5675 // (0.91751397f +
5676 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5677 //
5678 // error 0.00019228036, which is better than 12 bits
5679 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5680 getF32Constant(DAG, 0x3d431f31, dl));
5681 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5682 getF32Constant(DAG, 0x3ea21fb2, dl));
5683 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5684 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5685 getF32Constant(DAG, 0x3f6ae232, dl));
5686 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5687 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5688 getF32Constant(DAG, 0x3f25f7c3, dl));
5689 } else { // LimitFloatPrecision <= 18
5690 // For floating-point precision of 18:
5691 //
5692 // Log10ofMantissa =
5693 // -0.84299375f +
5694 // (1.5327582f +
5695 // (-1.0688956f +
5696 // (0.49102474f +
5697 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5698 //
5699 // error 0.0000037995730, which is better than 18 bits
5700 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5701 getF32Constant(DAG, 0x3c5d51ce, dl));
5702 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5703 getF32Constant(DAG, 0x3e00685a, dl));
5704 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5705 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5706 getF32Constant(DAG, 0x3efb6798, dl));
5707 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5708 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5709 getF32Constant(DAG, 0x3f88d192, dl));
5710 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5711 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5712 getF32Constant(DAG, 0x3fc4316c, dl));
5713 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5714 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5715 getF32Constant(DAG, 0x3f57ce70, dl));
5716 }
5717
5718 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5719 }
5720
5721 // No special expansion.
5722 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5723}
5724
5725/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5726/// limited-precision mode.
5728 const TargetLowering &TLI, SDNodeFlags Flags) {
5729 if (Op.getValueType() == MVT::f32 &&
5731 return getLimitedPrecisionExp2(Op, dl, DAG);
5732
5733 // No special expansion.
5734 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5735}
5736
5737/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5738/// limited-precision mode with x == 10.0f.
5739static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5740 SelectionDAG &DAG, const TargetLowering &TLI,
5741 SDNodeFlags Flags) {
5742 bool IsExp10 = false;
5743 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5745 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5746 APFloat Ten(10.0f);
5747 IsExp10 = LHSC->isExactlyValue(Ten);
5748 }
5749 }
5750
5751 // TODO: What fast-math-flags should be set on the FMUL node?
5752 if (IsExp10) {
5753 // Put the exponent in the right bit position for later addition to the
5754 // final result:
5755 //
5756 // #define LOG2OF10 3.3219281f
5757 // t0 = Op * LOG2OF10;
5758 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5759 getF32Constant(DAG, 0x40549a78, dl));
5760 return getLimitedPrecisionExp2(t0, dl, DAG);
5761 }
5762
5763 // No special expansion.
5764 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5765}
5766
5767/// ExpandPowI - Expand a llvm.powi intrinsic.
5768static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5769 SelectionDAG &DAG) {
5770 // If RHS is a constant, we can expand this out to a multiplication tree if
5771 // it's beneficial on the target, otherwise we end up lowering to a call to
5772 // __powidf2 (for example).
5773 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5774 unsigned Val = RHSC->getSExtValue();
5775
5776 // powi(x, 0) -> 1.0
5777 if (Val == 0)
5778 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5779
5781 Val, DAG.shouldOptForSize())) {
5782 // Get the exponent as a positive value.
5783 if ((int)Val < 0)
5784 Val = -Val;
5785 // We use the simple binary decomposition method to generate the multiply
5786 // sequence. There are more optimal ways to do this (for example,
5787 // powi(x,15) generates one more multiply than it should), but this has
5788 // the benefit of being both really simple and much better than a libcall.
5789 SDValue Res; // Logically starts equal to 1.0
5790 SDValue CurSquare = LHS;
5791 // TODO: Intrinsics should have fast-math-flags that propagate to these
5792 // nodes.
5793 while (Val) {
5794 if (Val & 1) {
5795 if (Res.getNode())
5796 Res =
5797 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5798 else
5799 Res = CurSquare; // 1.0*CurSquare.
5800 }
5801
5802 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5803 CurSquare, CurSquare);
5804 Val >>= 1;
5805 }
5806
5807 // If the original was negative, invert the result, producing 1/(x*x*x).
5808 if (RHSC->getSExtValue() < 0)
5809 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5810 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5811 return Res;
5812 }
5813 }
5814
5815 // Otherwise, expand to a libcall.
5816 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5817}
5818
5819static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5820 SDValue LHS, SDValue RHS, SDValue Scale,
5821 SelectionDAG &DAG, const TargetLowering &TLI) {
5822 EVT VT = LHS.getValueType();
5823 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5824 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5825 LLVMContext &Ctx = *DAG.getContext();
5826
5827 // If the type is legal but the operation isn't, this node might survive all
5828 // the way to operation legalization. If we end up there and we do not have
5829 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5830 // node.
5831
5832 // Coax the legalizer into expanding the node during type legalization instead
5833 // by bumping the size by one bit. This will force it to Promote, enabling the
5834 // early expansion and avoiding the need to expand later.
5835
5836 // We don't have to do this if Scale is 0; that can always be expanded, unless
5837 // it's a saturating signed operation. Those can experience true integer
5838 // division overflow, a case which we must avoid.
5839
5840 // FIXME: We wouldn't have to do this (or any of the early
5841 // expansion/promotion) if it was possible to expand a libcall of an
5842 // illegal type during operation legalization. But it's not, so things
5843 // get a bit hacky.
5844 unsigned ScaleInt = Scale->getAsZExtVal();
5845 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5846 (TLI.isTypeLegal(VT) ||
5847 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5849 Opcode, VT, ScaleInt);
5850 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5851 EVT PromVT;
5852 if (VT.isScalarInteger())
5853 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5854 else if (VT.isVector()) {
5855 PromVT = VT.getVectorElementType();
5856 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5857 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5858 } else
5859 llvm_unreachable("Wrong VT for DIVFIX?");
5860 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
5861 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
5862 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5863 // For saturating operations, we need to shift up the LHS to get the
5864 // proper saturation width, and then shift down again afterwards.
5865 if (Saturating)
5866 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5867 DAG.getConstant(1, DL, ShiftTy));
5868 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5869 if (Saturating)
5870 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5871 DAG.getConstant(1, DL, ShiftTy));
5872 return DAG.getZExtOrTrunc(Res, DL, VT);
5873 }
5874 }
5875
5876 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5877}
5878
5879// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5880// bitcasted, or split argument. Returns a list of <Register, size in bits>
5881static void
5882getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
5883 const SDValue &N) {
5884 switch (N.getOpcode()) {
5885 case ISD::CopyFromReg: {
5886 SDValue Op = N.getOperand(1);
5887 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5888 Op.getValueType().getSizeInBits());
5889 return;
5890 }
5891 case ISD::BITCAST:
5892 case ISD::AssertZext:
5893 case ISD::AssertSext:
5894 case ISD::TRUNCATE:
5895 getUnderlyingArgRegs(Regs, N.getOperand(0));
5896 return;
5897 case ISD::BUILD_PAIR:
5898 case ISD::BUILD_VECTOR:
5900 for (SDValue Op : N->op_values())
5901 getUnderlyingArgRegs(Regs, Op);
5902 return;
5903 default:
5904 return;
5905 }
5906}
5907
5908/// If the DbgValueInst is a dbg_value of a function argument, create the
5909/// corresponding DBG_VALUE machine instruction for it now. At the end of
5910/// instruction selection, they will be inserted to the entry BB.
5911/// We don't currently support this for variadic dbg_values, as they shouldn't
5912/// appear for function arguments or in the prologue.
5913bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
5914 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
5915 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
5916 const Argument *Arg = dyn_cast<Argument>(V);
5917 if (!Arg)
5918 return false;
5919
5922
5923 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
5924 // we've been asked to pursue.
5925 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
5926 bool Indirect) {
5927 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
5928 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
5929 // pointing at the VReg, which will be patched up later.
5930 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
5932 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
5933 /* isKill */ false, /* isDead */ false,
5934 /* isUndef */ false, /* isEarlyClobber */ false,
5935 /* SubReg */ 0, /* isDebug */ true)});
5936
5937 auto *NewDIExpr = FragExpr;
5938 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
5939 // the DIExpression.
5940 if (Indirect)
5941 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
5943 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
5944 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
5945 } else {
5946 // Create a completely standard DBG_VALUE.
5947 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
5948 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
5949 }
5950 };
5951
5952 if (Kind == FuncArgumentDbgValueKind::Value) {
5953 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5954 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
5955 // the entry block.
5956 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
5957 if (!IsInEntryBlock)
5958 return false;
5959
5960 // ArgDbgValues are hoisted to the beginning of the entry block. So we
5961 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
5962 // variable that also is a param.
5963 //
5964 // Although, if we are at the top of the entry block already, we can still
5965 // emit using ArgDbgValue. This might catch some situations when the
5966 // dbg.value refers to an argument that isn't used in the entry block, so
5967 // any CopyToReg node would be optimized out and the only way to express
5968 // this DBG_VALUE is by using the physical reg (or FI) as done in this
5969 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
5970 // we should only emit as ArgDbgValue if the Variable is an argument to the
5971 // current function, and the dbg.value intrinsic is found in the entry
5972 // block.
5973 bool VariableIsFunctionInputArg = Variable->isParameter() &&
5974 !DL->getInlinedAt();
5975 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
5976 if (!IsInPrologue && !VariableIsFunctionInputArg)
5977 return false;
5978
5979 // Here we assume that a function argument on IR level only can be used to
5980 // describe one input parameter on source level. If we for example have
5981 // source code like this
5982 //
5983 // struct A { long x, y; };
5984 // void foo(struct A a, long b) {
5985 // ...
5986 // b = a.x;
5987 // ...
5988 // }
5989 //
5990 // and IR like this
5991 //
5992 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
5993 // entry:
5994 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
5995 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
5996 // call void @llvm.dbg.value(metadata i32 %b, "b",
5997 // ...
5998 // call void @llvm.dbg.value(metadata i32 %a1, "b"
5999 // ...
6000 //
6001 // then the last dbg.value is describing a parameter "b" using a value that
6002 // is an argument. But since we already has used %a1 to describe a parameter
6003 // we should not handle that last dbg.value here (that would result in an
6004 // incorrect hoisting of the DBG_VALUE to the function entry).
6005 // Notice that we allow one dbg.value per IR level argument, to accommodate
6006 // for the situation with fragments above.
6007 if (VariableIsFunctionInputArg) {
6008 unsigned ArgNo = Arg->getArgNo();
6009 if (ArgNo >= FuncInfo.DescribedArgs.size())
6010 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6011 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6012 return false;
6013 FuncInfo.DescribedArgs.set(ArgNo);
6014 }
6015 }
6016
6017 bool IsIndirect = false;
6018 std::optional<MachineOperand> Op;
6019 // Some arguments' frame index is recorded during argument lowering.
6020 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6021 if (FI != std::numeric_limits<int>::max())
6023
6025 if (!Op && N.getNode()) {
6026 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6027 Register Reg;
6028 if (ArgRegsAndSizes.size() == 1)
6029 Reg = ArgRegsAndSizes.front().first;
6030
6031 if (Reg && Reg.isVirtual()) {
6033 Register PR = RegInfo.getLiveInPhysReg(Reg);
6034 if (PR)
6035 Reg = PR;
6036 }
6037 if (Reg) {
6038 Op = MachineOperand::CreateReg(Reg, false);
6039 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6040 }
6041 }
6042
6043 if (!Op && N.getNode()) {
6044 // Check if frame index is available.
6045 SDValue LCandidate = peekThroughBitcasts(N);
6046 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6047 if (FrameIndexSDNode *FINode =
6048 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6049 Op = MachineOperand::CreateFI(FINode->getIndex());
6050 }
6051
6052 if (!Op) {
6053 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6054 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
6055 SplitRegs) {
6056 unsigned Offset = 0;
6057 for (const auto &RegAndSize : SplitRegs) {
6058 // If the expression is already a fragment, the current register
6059 // offset+size might extend beyond the fragment. In this case, only
6060 // the register bits that are inside the fragment are relevant.
6061 int RegFragmentSizeInBits = RegAndSize.second;
6062 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6063 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6064 // The register is entirely outside the expression fragment,
6065 // so is irrelevant for debug info.
6066 if (Offset >= ExprFragmentSizeInBits)
6067 break;
6068 // The register is partially outside the expression fragment, only
6069 // the low bits within the fragment are relevant for debug info.
6070 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6071 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6072 }
6073 }
6074
6075 auto FragmentExpr = DIExpression::createFragmentExpression(
6076 Expr, Offset, RegFragmentSizeInBits);
6077 Offset += RegAndSize.second;
6078 // If a valid fragment expression cannot be created, the variable's
6079 // correct value cannot be determined and so it is set as Undef.
6080 if (!FragmentExpr) {
6082 Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
6083 DAG.AddDbgValue(SDV, false);
6084 continue;
6085 }
6086 MachineInstr *NewMI =
6087 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6088 Kind != FuncArgumentDbgValueKind::Value);
6089 FuncInfo.ArgDbgValues.push_back(NewMI);
6090 }
6091 };
6092
6093 // Check if ValueMap has reg number.
6095 VMI = FuncInfo.ValueMap.find(V);
6096 if (VMI != FuncInfo.ValueMap.end()) {
6097 const auto &TLI = DAG.getTargetLoweringInfo();
6098 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6099 V->getType(), std::nullopt);
6100 if (RFV.occupiesMultipleRegs()) {
6101 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6102 return true;
6103 }
6104
6105 Op = MachineOperand::CreateReg(VMI->second, false);
6106 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6107 } else if (ArgRegsAndSizes.size() > 1) {
6108 // This was split due to the calling convention, and no virtual register
6109 // mapping exists for the value.
6110 splitMultiRegDbgValue(ArgRegsAndSizes);
6111 return true;
6112 }
6113 }
6114
6115 if (!Op)
6116 return false;
6117
6119 "Expected inlined-at fields to agree");
6120 MachineInstr *NewMI = nullptr;
6121
6122 if (Op->isReg())
6123 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6124 else
6125 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6126 Variable, Expr);
6127
6128 // Otherwise, use ArgDbgValues.
6129 FuncInfo.ArgDbgValues.push_back(NewMI);
6130 return true;
6131}
6132
6133/// Return the appropriate SDDbgValue based on N.
6134SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6135 DILocalVariable *Variable,
6136 DIExpression *Expr,
6137 const DebugLoc &dl,
6138 unsigned DbgSDNodeOrder) {
6139 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6140 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6141 // stack slot locations.
6142 //
6143 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6144 // debug values here after optimization:
6145 //
6146 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6147 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6148 //
6149 // Both describe the direct values of their associated variables.
6150 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6151 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6152 }
6153 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6154 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6155}
6156
6157static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6158 switch (Intrinsic) {
6159 case Intrinsic::smul_fix:
6160 return ISD::SMULFIX;
6161 case Intrinsic::umul_fix:
6162 return ISD::UMULFIX;
6163 case Intrinsic::smul_fix_sat:
6164 return ISD::SMULFIXSAT;
6165 case Intrinsic::umul_fix_sat:
6166 return ISD::UMULFIXSAT;
6167 case Intrinsic::sdiv_fix:
6168 return ISD::SDIVFIX;
6169 case Intrinsic::udiv_fix:
6170 return ISD::UDIVFIX;
6171 case Intrinsic::sdiv_fix_sat:
6172 return ISD::SDIVFIXSAT;
6173 case Intrinsic::udiv_fix_sat:
6174 return ISD::UDIVFIXSAT;
6175 default:
6176 llvm_unreachable("Unhandled fixed point intrinsic");
6177 }
6178}
6179
6180void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
6181 const char *FunctionName) {
6182 assert(FunctionName && "FunctionName must not be nullptr");
6184 FunctionName,
6186 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
6187}
6188
6189/// Given a @llvm.call.preallocated.setup, return the corresponding
6190/// preallocated call.
6191static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6192 assert(cast<CallBase>(PreallocatedSetup)
6194 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6195 "expected call_preallocated_setup Value");
6196 for (const auto *U : PreallocatedSetup->users()) {
6197 auto *UseCall = cast<CallBase>(U);
6198 const Function *Fn = UseCall->getCalledFunction();
6199 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6200 return UseCall;
6201 }
6202 }
6203 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6204}
6205
6206/// If DI is a debug value with an EntryValue expression, lower it using the
6207/// corresponding physical register of the associated Argument value
6208/// (guaranteed to exist by the verifier).
6209bool SelectionDAGBuilder::visitEntryValueDbgValue(
6210 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6211 DIExpression *Expr, DebugLoc DbgLoc) {
6212 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6213 return false;
6214
6215 // These properties are guaranteed by the verifier.
6216 const Argument *Arg = cast<Argument>(Values[0]);
6217 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6218
6219 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6220 if (ArgIt == FuncInfo.ValueMap.end()) {
6221 LLVM_DEBUG(
6222 dbgs() << "Dropping dbg.value: expression is entry_value but "
6223 "couldn't find an associated register for the Argument\n");
6224 return true;
6225 }
6226 Register ArgVReg = ArgIt->getSecond();
6227
6228 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6229 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6231 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6232 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6233 return true;
6234 }
6235 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6236 "couldn't find a physical register\n");
6237 return true;
6238}
6239
6240/// Lower the call to the specified intrinsic function.
6241void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6242 unsigned Intrinsic) {
6243 SDLoc sdl = getCurSDLoc();
6244 switch (Intrinsic) {
6245 case Intrinsic::experimental_convergence_anchor:
6246 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6247 break;
6248 case Intrinsic::experimental_convergence_entry:
6249 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6250 break;
6251 case Intrinsic::experimental_convergence_loop: {
6252 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6253 auto *Token = Bundle->Inputs[0].get();
6254 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6255 getValue(Token)));
6256 break;
6257 }
6258 }
6259}
6260
6261/// Lower the call to the specified intrinsic function.
6262void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6263 unsigned Intrinsic) {
6265 SDLoc sdl = getCurSDLoc();
6266 DebugLoc dl = getCurDebugLoc();
6267 SDValue Res;
6268
6270 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6271 Flags.copyFMF(*FPOp);
6272
6273 switch (Intrinsic) {
6274 default:
6275 // By default, turn this into a target intrinsic node.
6276 visitTargetIntrinsic(I, Intrinsic);
6277 return;
6278 case Intrinsic::vscale: {
6279 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6280 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6281 return;
6282 }
6283 case Intrinsic::vastart: visitVAStart(I); return;
6284 case Intrinsic::vaend: visitVAEnd(I); return;
6285 case Intrinsic::vacopy: visitVACopy(I); return;
6286 case Intrinsic::returnaddress:
6288 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6289 getValue(I.getArgOperand(0))));
6290 return;
6291 case Intrinsic::addressofreturnaddress:
6292 setValue(&I,
6294 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6295 return;
6296 case Intrinsic::sponentry:
6297 setValue(&I,
6299 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6300 return;
6301 case Intrinsic::frameaddress:
6304 getValue(I.getArgOperand(0))));
6305 return;
6306 case Intrinsic::read_volatile_register:
6307 case Intrinsic::read_register: {
6308 Value *Reg = I.getArgOperand(0);
6309 SDValue Chain = getRoot();
6311 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6312 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6313 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6314 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6315 setValue(&I, Res);
6316 DAG.setRoot(Res.getValue(1));
6317 return;
6318 }
6319 case Intrinsic::write_register: {
6320 Value *Reg = I.getArgOperand(0);
6321 Value *RegValue = I.getArgOperand(1);
6322 SDValue Chain = getRoot();
6324 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6325 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6326 RegName, getValue(RegValue)));
6327 return;
6328 }
6329 case Intrinsic::memcpy: {
6330 const auto &MCI = cast<MemCpyInst>(I);
6331 SDValue Op1 = getValue(I.getArgOperand(0));
6332 SDValue Op2 = getValue(I.getArgOperand(1));
6333 SDValue Op3 = getValue(I.getArgOperand(2));
6334 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
6335 Align DstAlign = MCI.getDestAlign().valueOrOne();
6336 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6337 Align Alignment = std::min(DstAlign, SrcAlign);
6338 bool isVol = MCI.isVolatile();
6339 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6340 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6341 // node.
6342 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6343 SDValue MC = DAG.getMemcpy(
6344 Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6345 /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
6346 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
6347 updateDAGForMaybeTailCall(MC);
6348 return;
6349 }
6350 case Intrinsic::memcpy_inline: {
6351 const auto &MCI = cast<MemCpyInlineInst>(I);
6352 SDValue Dst = getValue(I.getArgOperand(0));
6353 SDValue Src = getValue(I.getArgOperand(1));
6354 SDValue Size = getValue(I.getArgOperand(2));
6355 assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
6356 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6357 Align DstAlign = MCI.getDestAlign().valueOrOne();
6358 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6359 Align Alignment = std::min(DstAlign, SrcAlign);
6360 bool isVol = MCI.isVolatile();
6361 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6362 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6363 // node.
6364 SDValue MC = DAG.getMemcpy(
6365 getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
6366 /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
6367 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
6368 updateDAGForMaybeTailCall(MC);
6369 return;
6370 }
6371 case Intrinsic::memset: {
6372 const auto &MSI = cast<MemSetInst>(I);
6373 SDValue Op1 = getValue(I.getArgOperand(0));
6374 SDValue Op2 = getValue(I.getArgOperand(1));
6375 SDValue Op3 = getValue(I.getArgOperand(2));
6376 // @llvm.memset defines 0 and 1 to both mean no alignment.
6377 Align Alignment = MSI.getDestAlign().valueOrOne();
6378 bool isVol = MSI.isVolatile();
6379 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6380 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6381 SDValue MS = DAG.getMemset(
6382 Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
6383 isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6384 updateDAGForMaybeTailCall(MS);
6385 return;
6386 }
6387 case Intrinsic::memset_inline: {
6388 const auto &MSII = cast<MemSetInlineInst>(I);
6389 SDValue Dst = getValue(I.getArgOperand(0));
6390 SDValue Value = getValue(I.getArgOperand(1));
6391 SDValue Size = getValue(I.getArgOperand(2));
6392 assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
6393 // @llvm.memset defines 0 and 1 to both mean no alignment.
6394 Align DstAlign = MSII.getDestAlign().valueOrOne();
6395 bool isVol = MSII.isVolatile();
6396 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6397 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6398 SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
6399 /* AlwaysInline */ true, isTC,
6400 MachinePointerInfo(I.getArgOperand(0)),
6401 I.getAAMetadata());
6402 updateDAGForMaybeTailCall(MC);
6403 return;
6404 }
6405 case Intrinsic::memmove: {
6406 const auto &MMI = cast<MemMoveInst>(I);
6407 SDValue Op1 = getValue(I.getArgOperand(0));
6408 SDValue Op2 = getValue(I.getArgOperand(1));
6409 SDValue Op3 = getValue(I.getArgOperand(2));
6410 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6411 Align DstAlign = MMI.getDestAlign().valueOrOne();
6412 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6413 Align Alignment = std::min(DstAlign, SrcAlign);
6414 bool isVol = MMI.isVolatile();
6415 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6416 // FIXME: Support passing different dest/src alignments to the memmove DAG
6417 // node.
6418 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6419 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6420 isTC, MachinePointerInfo(I.getArgOperand(0)),
6421 MachinePointerInfo(I.getArgOperand(1)),
6422 I.getAAMetadata(), AA);
6423 updateDAGForMaybeTailCall(MM);
6424 return;
6425 }
6426 case Intrinsic::memcpy_element_unordered_atomic: {
6427 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
6428 SDValue Dst = getValue(MI.getRawDest());
6429 SDValue Src = getValue(MI.getRawSource());
6430 SDValue Length = getValue(MI.getLength());
6431
6432 Type *LengthTy = MI.getLength()->getType();
6433 unsigned ElemSz = MI.getElementSizeInBytes();
6434 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6435 SDValue MC =
6436 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6437 isTC, MachinePointerInfo(MI.getRawDest()),
6438 MachinePointerInfo(MI.getRawSource()));
6439 updateDAGForMaybeTailCall(MC);
6440 return;
6441 }
6442 case Intrinsic::memmove_element_unordered_atomic: {
6443 auto &MI = cast<AtomicMemMoveInst>(I);
6444 SDValue Dst = getValue(MI.getRawDest());
6445 SDValue Src = getValue(MI.getRawSource());
6446 SDValue Length = getValue(MI.getLength());
6447
6448 Type *LengthTy = MI.getLength()->getType();
6449 unsigned ElemSz = MI.getElementSizeInBytes();
6450 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6451 SDValue MC =
6452 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6453 isTC, MachinePointerInfo(MI.getRawDest()),
6454 MachinePointerInfo(MI.getRawSource()));
6455 updateDAGForMaybeTailCall(MC);
6456 return;
6457 }
6458 case Intrinsic::memset_element_unordered_atomic: {
6459 auto &MI = cast<AtomicMemSetInst>(I);
6460 SDValue Dst = getValue(MI.getRawDest());
6461 SDValue Val = getValue(MI.getValue());
6462 SDValue Length = getValue(MI.getLength());
6463
6464 Type *LengthTy = MI.getLength()->getType();
6465 unsigned ElemSz = MI.getElementSizeInBytes();
6466 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6467 SDValue MC =
6468 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
6469 isTC, MachinePointerInfo(MI.getRawDest()));
6470 updateDAGForMaybeTailCall(MC);
6471 return;
6472 }
6473 case Intrinsic::call_preallocated_setup: {
6474 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6475 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6476 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6477 getRoot(), SrcValue);
6478 setValue(&I, Res);
6479 DAG.setRoot(Res);
6480 return;
6481 }
6482 case Intrinsic::call_preallocated_arg: {
6483 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6484 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6485 SDValue Ops[3];
6486 Ops[0] = getRoot();
6487 Ops[1] = SrcValue;
6488 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6489 MVT::i32); // arg index
6490 SDValue Res = DAG.getNode(
6492 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6493 setValue(&I, Res);
6494 DAG.setRoot(Res.getValue(1));
6495 return;
6496 }
6497 case Intrinsic::dbg_declare: {
6498 const auto &DI = cast<DbgDeclareInst>(I);
6499 // Debug intrinsics are handled separately in assignment tracking mode.
6500 // Some intrinsics are handled right after Argument lowering.
6501 if (AssignmentTrackingEnabled ||
6503 return;
6504 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DI << "\n");
6505 DILocalVariable *Variable = DI.getVariable();
6506 DIExpression *Expression = DI.getExpression();
6508 // Assume dbg.declare can not currently use DIArgList, i.e.
6509 // it is non-variadic.
6510 assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6511 handleDebugDeclare(DI.getVariableLocationOp(0), Variable, Expression,
6512 DI.getDebugLoc());
6513 return;
6514 }
6515 case Intrinsic::dbg_label: {
6516 const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6517 DILabel *Label = DI.getLabel();
6518 assert(Label && "Missing label");
6519
6520 SDDbgLabel *SDV;
6521 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6522 DAG.AddDbgLabel(SDV);
6523 return;
6524 }
6525 case Intrinsic::dbg_assign: {
6526 // Debug intrinsics are handled seperately in assignment tracking mode.
6527 if (AssignmentTrackingEnabled)
6528 return;
6529 // If assignment tracking hasn't been enabled then fall through and treat
6530 // the dbg.assign as a dbg.value.
6531 [[fallthrough]];
6532 }
6533 case Intrinsic::dbg_value: {
6534 // Debug intrinsics are handled seperately in assignment tracking mode.
6535 if (AssignmentTrackingEnabled)
6536 return;
6537 const DbgValueInst &DI = cast<DbgValueInst>(I);
6538 assert(DI.getVariable() && "Missing variable");
6539
6540 DILocalVariable *Variable = DI.getVariable();
6543
6544 if (DI.isKillLocation()) {
6545 handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
6546 return;
6547 }
6548
6550 if (Values.empty())
6551 return;
6552
6553 bool IsVariadic = DI.hasArgList();
6554 if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
6555 SDNodeOrder, IsVariadic))
6556 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
6557 DI.getDebugLoc(), SDNodeOrder);
6558 return;
6559 }
6560
6561 case Intrinsic::eh_typeid_for: {
6562 // Find the type id for the given typeinfo.
6563 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6564 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6565 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6566 setValue(&I, Res);
6567 return;
6568 }
6569
6570 case Intrinsic::eh_return_i32:
6571 case Intrinsic::eh_return_i64:
6574 MVT::Other,
6576 getValue(I.getArgOperand(0)),
6577 getValue(I.getArgOperand(1))));
6578 return;
6579 case Intrinsic::eh_unwind_init:
6581 return;
6582 case Intrinsic::eh_dwarf_cfa:
6585 getValue(I.getArgOperand(0))));
6586 return;
6587 case Intrinsic::eh_sjlj_callsite: {
6589 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6590 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!");
6591
6593 return;
6594 }
6595 case Intrinsic::eh_sjlj_functioncontext: {
6596 // Get and store the index of the function context.
6598 AllocaInst *FnCtx =
6599 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6600 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6602 return;
6603 }
6604 case Intrinsic::eh_sjlj_setjmp: {
6605 SDValue Ops[2];
6606 Ops[0] = getRoot();
6607 Ops[1] = getValue(I.getArgOperand(0));
6609 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6610 setValue(&I, Op.getValue(0));
6611 DAG.setRoot(Op.getValue(1));
6612 return;
6613 }
6614 case Intrinsic::eh_sjlj_longjmp:
6615 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6616 getRoot(), getValue(I.getArgOperand(0))));
6617 return;
6618 case Intrinsic::eh_sjlj_setup_dispatch:
6620 getRoot()));
6621 return;
6622 case Intrinsic::masked_gather:
6623 visitMaskedGather(I);
6624 return;
6625 case Intrinsic::masked_load:
6626 visitMaskedLoad(I);
6627 return;
6628 case Intrinsic::masked_scatter:
6629 visitMaskedScatter(I);
6630 return;
6631 case Intrinsic::masked_store:
6632 visitMaskedStore(I);
6633 return;
6634 case Intrinsic::masked_expandload:
6635 visitMaskedLoad(I, true /* IsExpanding */);
6636 return;
6637 case Intrinsic::masked_compressstore:
6638 visitMaskedStore(I, true /* IsCompressing */);
6639 return;
6640 case Intrinsic::powi:
6641 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6642 getValue(I.getArgOperand(1)), DAG));
6643 return;
6644 case Intrinsic::log:
6645 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6646 return;
6647 case Intrinsic::log2:
6648 setValue(&I,
6649 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6650 return;
6651 case Intrinsic::log10:
6652 setValue(&I,
6653 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6654 return;
6655 case Intrinsic::exp:
6656 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6657 return;
6658 case Intrinsic::exp2:
6659 setValue(&I,
6660 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6661 return;
6662 case Intrinsic::pow:
6663 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6664 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6665 return;
6666 case Intrinsic::sqrt:
6667 case Intrinsic::fabs:
6668 case Intrinsic::sin:
6669 case Intrinsic::cos:
6670 case Intrinsic::exp10:
6671 case Intrinsic::floor:
6672 case Intrinsic::ceil:
6673 case Intrinsic::trunc:
6674 case Intrinsic::rint:
6675 case Intrinsic::nearbyint:
6676 case Intrinsic::round:
6677 case Intrinsic::roundeven:
6678 case Intrinsic::canonicalize: {
6679 unsigned Opcode;
6680 switch (Intrinsic) {
6681 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6682 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6683 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6684 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6685 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6686 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6687 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6688 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6689 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6690 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6691 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6692 case Intrinsic::round: Opcode = ISD::FROUND; break;
6693 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6694 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6695 }
6696
6697 setValue(&I, DAG.getNode(Opcode, sdl,
6698 getValue(I.getArgOperand(0)).getValueType(),
6699 getValue(I.getArgOperand(0)), Flags));
6700 return;
6701 }
6702 case Intrinsic::lround:
6703 case Intrinsic::llround:
6704 case Intrinsic::lrint:
6705 case Intrinsic::llrint: {
6706 unsigned Opcode;
6707 switch (Intrinsic) {
6708 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6709 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6710 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6711 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6712 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6713 }
6714
6715 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6716 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6717 getValue(I.getArgOperand(0))));
6718 return;
6719 }
6720 case Intrinsic::minnum:
6722 getValue(I.getArgOperand(0)).getValueType(),
6723 getValue(I.getArgOperand(0)),
6724 getValue(I.getArgOperand(1)), Flags));
6725 return;
6726 case Intrinsic::maxnum:
6728 getValue(I.getArgOperand(0)).getValueType(),
6729 getValue(I.getArgOperand(0)),
6730 getValue(I.getArgOperand(1)), Flags));
6731 return;
6732 case Intrinsic::minimum:
6734 getValue(I.getArgOperand(0)).getValueType(),
6735 getValue(I.getArgOperand(0)),
6736 getValue(I.getArgOperand(1)), Flags));
6737 return;
6738 case Intrinsic::maximum:
6740 getValue(I.getArgOperand(0)).getValueType(),
6741 getValue(I.getArgOperand(0)),
6742 getValue(I.getArgOperand(1)), Flags));
6743 return;
6744 case Intrinsic::copysign:
6746 getValue(I.getArgOperand(0)).getValueType(),
6747 getValue(I.getArgOperand(0)),
6748 getValue(I.getArgOperand(1)), Flags));
6749 return;
6750 case Intrinsic::ldexp:
6752 getValue(I.getArgOperand(0)).getValueType(),
6753 getValue(I.getArgOperand(0)),
6754 getValue(I.getArgOperand(1)), Flags));
6755 return;
6756 case Intrinsic::frexp: {
6757 SmallVector<EVT, 2> ValueVTs;
6758 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
6759 SDVTList VTs = DAG.getVTList(ValueVTs);
6760 setValue(&I,
6761 DAG.getNode(ISD::FFREXP, sdl, VTs, getValue(I.getArgOperand(0))));
6762 return;
6763 }
6764 case Intrinsic::arithmetic_fence: {
6766 getValue(I.getArgOperand(0)).getValueType(),
6767 getValue(I.getArgOperand(0)), Flags));
6768 return;
6769 }
6770 case Intrinsic::fma:
6772 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
6773 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
6774 getValue(I.getArgOperand(2)), Flags));
6775 return;
6776#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
6777 case Intrinsic::INTRINSIC:
6778#include "llvm/IR/ConstrainedOps.def"
6779 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
6780 return;
6781#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6782#include "llvm/IR/VPIntrinsics.def"
6783 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
6784 return;
6785 case Intrinsic::fptrunc_round: {
6786 // Get the last argument, the metadata and convert it to an integer in the
6787 // call
6788 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
6789 std::optional<RoundingMode> RoundMode =
6790 convertStrToRoundingMode(cast<MDString>(MD)->getString());
6791
6792 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6793
6794 // Propagate fast-math-flags from IR to node(s).
6796 Flags.copyFMF(*cast<FPMathOperator>(&I));
6797 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
6798
6800 Result = DAG.getNode(
6801 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
6802 DAG.getTargetConstant((int)*RoundMode, sdl,
6804 setValue(&I, Result);
6805
6806 return;
6807 }
6808 case Intrinsic::fmuladd: {
6809 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6812 setValue(&I, DAG.getNode(ISD::FMA, sdl,
6813 getValue(I.getArgOperand(0)).getValueType(),
6814 getValue(I.getArgOperand(0)),
6815 getValue(I.getArgOperand(1)),
6816 getValue(I.getArgOperand(2)), Flags));
6817 } else {
6818 // TODO: Intrinsic calls should have fast-math-flags.
6820 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
6821 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
6823 getValue(I.getArgOperand(0)).getValueType(),
6824 Mul, getValue(I.getArgOperand(2)), Flags);
6825 setValue(&I, Add);
6826 }
6827 return;
6828 }
6829 case Intrinsic::convert_to_fp16:
6830 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
6831 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
6832 getValue(I.getArgOperand(0)),
6833 DAG.getTargetConstant(0, sdl,
6834 MVT::i32))));
6835 return;
6836 case Intrinsic::convert_from_fp16:
6838 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6839 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
6840 getValue(I.getArgOperand(0)))));
6841 return;
6842 case Intrinsic::fptosi_sat: {
6843 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6845 getValue(I.getArgOperand(0)),
6847 return;
6848 }
6849 case Intrinsic::fptoui_sat: {
6850 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6852 getValue(I.getArgOperand(0)),
6854 return;
6855 }
6856 case Intrinsic::set_rounding:
6857 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
6858 {getRoot(), getValue(I.getArgOperand(0))});
6859 setValue(&I, Res);
6860 DAG.setRoot(Res.getValue(0));
6861 return;
6862 case Intrinsic::is_fpclass: {
6863 const DataLayout DLayout = DAG.getDataLayout();
6864 EVT DestVT = TLI.getValueType(DLayout, I.getType());
6865 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
6866 FPClassTest Test = static_cast<FPClassTest>(
6867 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
6869 const Function &F = MF.getFunction();
6870 SDValue Op = getValue(I.getArgOperand(0));
6872 Flags.setNoFPExcept(
6873 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
6874 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
6875 // expansion can use illegal types. Making expansion early allows
6876 // legalizing these types prior to selection.
6877 if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
6878 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
6879 setValue(&I, Result);
6880 return;
6881 }
6882
6883 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
6884 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
6885 setValue(&I, V);
6886 return;
6887 }
6888 case Intrinsic::get_fpenv: {
6889 const DataLayout DLayout = DAG.getDataLayout();
6890 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
6891 Align TempAlign = DAG.getEVTAlign(EnvVT);
6892 SDValue Chain = getRoot();
6893 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
6894 // and temporary storage in stack.
6895 if (TLI.isOperationLegalOrCustom(ISD::GET_FPENV, EnvVT)) {
6896 Res = DAG.getNode(
6897 ISD::GET_FPENV, sdl,
6898 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
6899 MVT::Other),
6900 Chain);
6901 } else {
6902 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
6903 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
6904 auto MPI =
6908 TempAlign);
6909 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
6910 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
6911 }
6912 setValue(&I, Res);
6913 DAG.setRoot(Res.getValue(1));
6914 return;
6915 }
6916 case Intrinsic::set_fpenv: {
6917 const DataLayout DLayout = DAG.getDataLayout();
6918 SDValue Env = getValue(I.getArgOperand(0));
6919 EVT EnvVT = Env.getValueType();
6920 Align TempAlign = DAG.getEVTAlign(EnvVT);
6921 SDValue Chain = getRoot();
6922 // If SET_FPENV is custom or legal, use it. Otherwise use loading
6923 // environment from memory.
6924 if (TLI.isOperationLegalOrCustom(ISD::SET_FPENV, EnvVT)) {
6925 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
6926 } else {
6927 // Allocate space in stack, copy environment bits into it and use this
6928 // memory in SET_FPENV_MEM.
6929 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
6930 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
6931 auto MPI =
6933 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
6937 TempAlign);
6938 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
6939 }
6940 DAG.setRoot(Chain);
6941 return;
6942 }
6943 case Intrinsic::reset_fpenv:
6944 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
6945 return;
6946 case Intrinsic::get_fpmode:
6947 Res = DAG.getNode(
6948 ISD::GET_FPMODE, sdl,
6949 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
6950 MVT::Other),
6951 DAG.getRoot());
6952 setValue(&I, Res);
6953 DAG.setRoot(Res.getValue(1));
6954 return;
6955 case Intrinsic::set_fpmode:
6956 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
6957 getValue(I.getArgOperand(0)));
6958 DAG.setRoot(Res);
6959 return;
6960 case Intrinsic::reset_fpmode: {
6961 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
6962 DAG.setRoot(Res);
6963 return;
6964 }
6965 case Intrinsic::pcmarker: {
6966 SDValue Tmp = getValue(I.getArgOperand(0));
6967 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
6968 return;
6969 }
6970 case Intrinsic::readcyclecounter: {
6971 SDValue Op = getRoot();
6973 DAG.getVTList(MVT::i64, MVT::Other), Op);
6974 setValue(&I, Res);
6975 DAG.setRoot(Res.getValue(1));
6976 return;
6977 }
6978 case Intrinsic::readsteadycounter: {
6979 SDValue Op = getRoot();
6981 DAG.getVTList(MVT::i64, MVT::Other), Op);
6982 setValue(&I, Res);
6983 DAG.setRoot(Res.getValue(1));
6984 return;
6985 }
6986 case Intrinsic::bitreverse:
6988 getValue(I.getArgOperand(0)).getValueType(),
6989 getValue(I.getArgOperand(0))));
6990 return;
6991 case Intrinsic::bswap:
6993 getValue(I.getArgOperand(0)).getValueType(),
6994 getValue(I.getArgOperand(0))));
6995 return;
6996 case Intrinsic::cttz: {
6997 SDValue Arg = getValue(I.getArgOperand(0));
6998 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
6999 EVT Ty = Arg.getValueType();
7001 sdl, Ty, Arg));
7002 return;
7003 }
7004 case Intrinsic::ctlz: {
7005 SDValue Arg = getValue(I.getArgOperand(0));
7006 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7007 EVT Ty = Arg.getValueType();
7009 sdl, Ty, Arg));
7010 return;
7011 }
7012 case Intrinsic::ctpop: {
7013 SDValue Arg = getValue(I.getArgOperand(0));
7014 EVT Ty = Arg.getValueType();
7015 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7016 return;
7017 }
7018 case Intrinsic::fshl:
7019 case Intrinsic::fshr: {
7020 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7021 SDValue X = getValue(I.getArgOperand(0));
7022 SDValue Y = getValue(I.getArgOperand(1));
7023 SDValue Z = getValue(I.getArgOperand(2));
7024 EVT VT = X.getValueType();
7025
7026 if (X == Y) {
7027 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7028 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7029 } else {
7030 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7031 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7032 }
7033 return;
7034 }
7035 case Intrinsic::sadd_sat: {
7036 SDValue Op1 = getValue(I.getArgOperand(0));
7037 SDValue Op2 = getValue(I.getArgOperand(1));
7038 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7039 return;
7040 }
7041 case Intrinsic::uadd_sat: {
7042 SDValue Op1 = getValue(I.getArgOperand(0));
7043 SDValue Op2 = getValue(I.getArgOperand(1));
7044 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7045 return;
7046 }
7047 case Intrinsic::ssub_sat: {
7048 SDValue Op1 = getValue(I.getArgOperand(0));
7049 SDValue Op2 = getValue(I.getArgOperand(1));
7050 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7051 return;
7052 }
7053 case Intrinsic::usub_sat: {
7054 SDValue Op1 = getValue(I.getArgOperand(0));
7055 SDValue Op2 = getValue(I.getArgOperand(1));
7056 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7057 return;
7058 }
7059 case Intrinsic::sshl_sat: {
7060 SDValue Op1 = getValue(I.getArgOperand(0));
7061 SDValue Op2 = getValue(I.getArgOperand(1));
7062 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7063 return;
7064 }
7065 case Intrinsic::ushl_sat: {
7066 SDValue Op1 = getValue(I.getArgOperand(0));
7067 SDValue Op2 = getValue(I.getArgOperand(1));
7068 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7069 return;
7070 }
7071 case Intrinsic::smul_fix:
7072 case Intrinsic::umul_fix:
7073 case Intrinsic::smul_fix_sat:
7074 case Intrinsic::umul_fix_sat: {
7075 SDValue Op1 = getValue(I.getArgOperand(0));
7076 SDValue Op2 = getValue(I.getArgOperand(1));
7077 SDValue Op3 = getValue(I.getArgOperand(2));
7079 Op1.getValueType(), Op1, Op2, Op3));
7080 return;
7081 }
7082 case Intrinsic::sdiv_fix:
7083 case Intrinsic::udiv_fix:
7084 case Intrinsic::sdiv_fix_sat:
7085 case Intrinsic::udiv_fix_sat: {
7086 SDValue Op1 = getValue(I.getArgOperand(0));
7087 SDValue Op2 = getValue(I.getArgOperand(1));
7088 SDValue Op3 = getValue(I.getArgOperand(2));
7090 Op1, Op2, Op3, DAG, TLI));
7091 return;
7092 }
7093 case Intrinsic::smax: {
7094 SDValue Op1 = getValue(I.getArgOperand(0));
7095 SDValue Op2 = getValue(I.getArgOperand(1));
7096 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7097 return;
7098 }
7099 case Intrinsic::smin: {
7100 SDValue Op1 = getValue(I.getArgOperand(0));
7101 SDValue Op2 = getValue(I.getArgOperand(1));
7102 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7103 return;
7104 }
7105 case Intrinsic::umax: {
7106 SDValue Op1 = getValue(I.getArgOperand(0));
7107 SDValue Op2 = getValue(I.getArgOperand(1));
7108 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7109 return;
7110 }
7111 case Intrinsic::umin: {
7112 SDValue Op1 = getValue(I.getArgOperand(0));
7113 SDValue Op2 = getValue(I.getArgOperand(1));
7114 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7115 return;
7116 }
7117 case Intrinsic::abs: {
7118 // TODO: Preserve "int min is poison" arg in SDAG?
7119 SDValue Op1 = getValue(I.getArgOperand(0));
7120 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
7121 return;
7122 }
7123 case Intrinsic::stacksave: {
7124 SDValue Op = getRoot();
7125 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7126 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
7127 setValue(&I, Res);
7128 DAG.setRoot(Res.getValue(1));
7129 return;
7130 }
7131 case Intrinsic::stackrestore:
7132 Res = getValue(I.getArgOperand(0));
7133 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7134 return;
7135 case Intrinsic::get_dynamic_area_offset: {
7136 SDValue Op = getRoot();
7137 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7138 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7139 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
7140 // target.
7141 if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
7142 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
7143 " intrinsic!");
7145 Op);
7146 DAG.setRoot(Op);
7147 setValue(&I, Res);
7148 return;
7149 }
7150 case Intrinsic::stackguard: {
7152 const Module &M = *MF.getFunction().getParent();
7153 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7154 SDValue Chain = getRoot();
7155 if (TLI.useLoadStackGuardNode()) {
7156 Res = getLoadStackGuard(DAG, sdl, Chain);
7157 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7158 } else {
7159 const Value *Global = TLI.getSDagStackGuard(M);
7161 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7164 }
7165 if (TLI.useStackGuardXorFP())
7166 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
7167 DAG.setRoot(Chain);
7168 setValue(&I, Res);
7169 return;
7170 }
7171 case Intrinsic::stackprotector: {
7172 // Emit code into the DAG to store the stack guard onto the stack.
7174 MachineFrameInfo &MFI = MF.getFrameInfo();
7175 SDValue Src, Chain = getRoot();
7176
7177 if (TLI.useLoadStackGuardNode())
7178 Src = getLoadStackGuard(DAG, sdl, Chain);
7179 else
7180 Src = getValue(I.getArgOperand(0)); // The guard's value.
7181
7182 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7183
7184 int FI = FuncInfo.StaticAllocaMap[Slot];
7185 MFI.setStackProtectorIndex(FI);
7186 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7187
7188 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7189
7190 // Store the stack protector onto the stack.
7191 Res = DAG.getStore(
7192 Chain, sdl, Src, FIN,
7195 setValue(&I, Res);
7196 DAG.setRoot(Res);
7197 return;
7198 }
7199 case Intrinsic::objectsize:
7200 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7201
7202 case Intrinsic::is_constant:
7203 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7204
7205 case Intrinsic::annotation:
7206 case Intrinsic::ptr_annotation:
7207 case Intrinsic::launder_invariant_group:
7208 case Intrinsic::strip_invariant_group:
7209 // Drop the intrinsic, but forward the value
7210 setValue(&I, getValue(I.getOperand(0)));
7211 return;
7212
7213 case Intrinsic::assume:
7214 case Intrinsic::experimental_noalias_scope_decl:
7215 case Intrinsic::var_annotation:
7216 case Intrinsic::sideeffect:
7217 // Discard annotate attributes, noalias scope declarations, assumptions, and
7218 // artificial side-effects.
7219 return;
7220
7221 case Intrinsic::codeview_annotation: {
7222 // Emit a label associated with this metadata.
7224 MCSymbol *Label =
7225 MF.getMMI().getContext().createTempSymbol("annotation", true);
7226 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7227 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7228 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7229 DAG.setRoot(Res);
7230 return;
7231 }
7232
7233 case Intrinsic::init_trampoline: {
7234 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7235
7236 SDValue Ops[6];
7237 Ops[0] = getRoot();
7238 Ops[1] = getValue(I.getArgOperand(0));
7239 Ops[2] = getValue(I.getArgOperand(1));
7240 Ops[3] = getValue(I.getArgOperand(2));
7241 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7242 Ops[5] = DAG.getSrcValue(F);
7243
7244 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7245
7246 DAG.setRoot(Res);
7247 return;
7248 }
7249 case Intrinsic::adjust_trampoline:
7252 getValue(I.getArgOperand(0))));
7253 return;
7254 case Intrinsic::gcroot: {
7256 "only valid in functions with gc specified, enforced by Verifier");
7257 assert(GFI && "implied by previous");
7258 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7259 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7260
7261 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7262 GFI->addStackRoot(FI->getIndex(), TypeMap);
7263 return;
7264 }
7265 case Intrinsic::gcread:
7266 case Intrinsic::gcwrite:
7267 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7268 case Intrinsic::get_rounding:
7269 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7270 setValue(&I, Res);
7271 DAG.setRoot(Res.getValue(1));
7272 return;
7273
7274 case Intrinsic::expect:
7275 // Just replace __builtin_expect(exp, c) with EXP.
7276 setValue(&I, getValue(I.getArgOperand(0)));
7277 return;
7278
7279 case Intrinsic::ubsantrap:
7280 case Intrinsic::debugtrap:
7281 case Intrinsic::trap: {
7282 StringRef TrapFuncName =
7283 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
7284 if (TrapFuncName.empty()) {
7285 switch (Intrinsic) {
7286 case Intrinsic::trap:
7287 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
7288 break;
7289 case Intrinsic::debugtrap:
7290 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
7291 break;
7292 case Intrinsic::ubsantrap:
7294 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
7296 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
7297 MVT::i32)));
7298 break;
7299 default: llvm_unreachable("unknown trap intrinsic");
7300 }
7301 return;
7302 }
7304 if (Intrinsic == Intrinsic::ubsantrap) {
7306 Args[0].Val = I.getArgOperand(0);
7307 Args[0].Node = getValue(Args[0].Val);
7308 Args[0].Ty = Args[0].Val->getType();
7309 }
7310
7312 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7313 CallingConv::C, I.getType(),
7314 DAG.getExternalSymbol(TrapFuncName.data(),
7316 std::move(Args));
7317
7318 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7319 DAG.setRoot(Result.second);
7320 return;
7321 }
7322
7323 case Intrinsic::uadd_with_overflow:
7324 case Intrinsic::sadd_with_overflow:
7325 case Intrinsic::usub_with_overflow:
7326 case Intrinsic::ssub_with_overflow:
7327 case Intrinsic::umul_with_overflow:
7328 case Intrinsic::smul_with_overflow: {
7330 switch (Intrinsic) {
7331 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7332 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7333 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7334 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7335 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7336 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7337 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7338 }
7339 SDValue Op1 = getValue(I.getArgOperand(0));
7340 SDValue Op2 = getValue(I.getArgOperand(1));
7341
7342 EVT ResultVT = Op1.getValueType();
7343 EVT OverflowVT = MVT::i1;
7344 if (ResultVT.isVector())
7345 OverflowVT = EVT::getVectorVT(
7346 *Context, OverflowVT, ResultVT.getVectorElementCount());
7347
7348 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
7349 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
7350 return;
7351 }
7352 case Intrinsic::prefetch: {
7353 SDValue Ops[5];
7354 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7356 Ops[0] = DAG.getRoot();
7357 Ops[1] = getValue(I.getArgOperand(0));
7358 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7359 MVT::i32);
7360 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
7361 MVT::i32);
7362 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
7363 MVT::i32);
7365 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
7366 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
7367 /* align */ std::nullopt, Flags);
7368
7369 // Chain the prefetch in parallel with any pending loads, to stay out of
7370 // the way of later optimizations.
7371 PendingLoads.push_back(Result);
7372 Result = getRoot();
7373 DAG.setRoot(Result);
7374 return;
7375 }
7376 case Intrinsic::lifetime_start:
7377 case Intrinsic::lifetime_end: {
7378 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7379 // Stack coloring is not enabled in O0, discard region information.
7381 return;
7382
7383 const int64_t ObjectSize =
7384 cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
7385 Value *const ObjectPtr = I.getArgOperand(1);
7387 getUnderlyingObjects(ObjectPtr, Allocas);
7388
7389 for (const Value *Alloca : Allocas) {
7390 const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
7391
7392 // Could not find an Alloca.
7393 if (!LifetimeObject)
7394 continue;
7395
7396 // First check that the Alloca is static, otherwise it won't have a
7397 // valid frame index.
7398 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
7399 if (SI == FuncInfo.StaticAllocaMap.end())
7400 return;
7401
7402 const int FrameIndex = SI->second;
7403 int64_t Offset;
7405 ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
7406 Offset = -1; // Cannot determine offset from alloca to lifetime object.
7407 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
7408 Offset);
7409 DAG.setRoot(Res);
7410 }
7411 return;
7412 }
7413 case Intrinsic::pseudoprobe: {
7414 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
7415 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7416 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
7417 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
7418 DAG.setRoot(Res);
7419 return;
7420 }
7421 case Intrinsic::invariant_start:
7422 // Discard region information.
7423 setValue(&I,
7424 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
7425 return;
7426 case Intrinsic::invariant_end:
7427 // Discard region information.
7428 return;
7429 case Intrinsic::clear_cache:
7430 /// FunctionName may be null.
7431 if (const char *FunctionName = TLI.getClearCacheBuiltinName())
7432 lowerCallToExternalSymbol(I, FunctionName);
7433 return;
7434 case Intrinsic::donothing:
7435 case Intrinsic::seh_try_begin:
7436 case Intrinsic::seh_scope_begin:
7437 case Intrinsic::seh_try_end:
7438 case Intrinsic::seh_scope_end:
7439 // ignore
7440 return;
7441 case Intrinsic::experimental_stackmap:
7442 visitStackmap(I);
7443 return;
7444 case Intrinsic::experimental_patchpoint_void:
7445 case Intrinsic::experimental_patchpoint_i64:
7446 visitPatchpoint(I);
7447 return;
7448 case Intrinsic::experimental_gc_statepoint:
7449 LowerStatepoint(cast<GCStatepointInst>(I));
7450 return;
7451 case Intrinsic::experimental_gc_result:
7452 visitGCResult(cast<GCResultInst>(I));
7453 return;
7454 case Intrinsic::experimental_gc_relocate:
7455 visitGCRelocate(cast<GCRelocateInst>(I));
7456 return;
7457 case Intrinsic::instrprof_cover:
7458 llvm_unreachable("instrprof failed to lower a cover");
7459 case Intrinsic::instrprof_increment:
7460 llvm_unreachable("instrprof failed to lower an increment");
7461 case Intrinsic::instrprof_timestamp:
7462 llvm_unreachable("instrprof failed to lower a timestamp");
7463 case Intrinsic::instrprof_value_profile:
7464 llvm_unreachable("instrprof failed to lower a value profiling call");
7465 case Intrinsic::instrprof_mcdc_parameters:
7466 llvm_unreachable("instrprof failed to lower mcdc parameters");
7467 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7468 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7469 case Intrinsic::instrprof_mcdc_condbitmap_update:
7470 llvm_unreachable("instrprof failed to lower an mcdc condbitmap update");
7471 case Intrinsic::localescape: {
7474
7475 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7476 // is the same on all targets.
7477 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7478 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
7479 if (isa<ConstantPointerNull>(Arg))
7480 continue; // Skip null pointers. They represent a hole in index space.
7481 AllocaInst *Slot = cast<AllocaInst>(Arg);
7482 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7483 "can only escape static allocas");
7484 int FI = FuncInfo.StaticAllocaMap[Slot];
7485 MCSymbol *FrameAllocSym =
7489 TII->get(TargetOpcode::LOCAL_ESCAPE))
7490 .addSym(FrameAllocSym)
7491 .addFrameIndex(FI);
7492 }
7493
7494 return;
7495 }
7496
7497 case Intrinsic::localrecover: {
7498 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7500
7501 // Get the symbol that defines the frame offset.
7502 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
7503 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
7504 unsigned IdxVal =
7505 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
7506 MCSymbol *FrameAllocSym =
7509
7510 Value *FP = I.getArgOperand(1);
7511 SDValue FPVal = getValue(FP);
7512 EVT PtrVT = FPVal.getValueType();
7513
7514 // Create a MCSymbol for the label to avoid any target lowering
7515 // that would make this PC relative.
7516 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
7517 SDValue OffsetVal =
7518 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
7519
7520 // Add the offset to the FP.
7521 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
7522 setValue(&I, Add);
7523
7524 return;
7525 }
7526
7527 case Intrinsic::eh_exceptionpointer:
7528 case Intrinsic::eh_exceptioncode: {
7529 // Get the exception pointer vreg, copy from it, and resize it to fit.
7530 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7531 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7532 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7533 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
7534 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7535 if (Intrinsic == Intrinsic::eh_exceptioncode)
7536 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7537 setValue(&I, N);
7538 return;
7539 }
7540 case Intrinsic::xray_customevent: {
7541 // Here we want to make sure that the intrinsic behaves as if it has a
7542 // specific calling convention.
7543 const auto &Triple = DAG.getTarget().getTargetTriple();
7544 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7545 return;
7546
7548
7549 // We want to say that we always want the arguments in registers.
7550 SDValue LogEntryVal = getValue(I.getArgOperand(0));
7551 SDValue StrSizeVal = getValue(I.getArgOperand(1));
7552 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7553 SDValue Chain = getRoot();
7554 Ops.push_back(LogEntryVal);
7555 Ops.push_back(StrSizeVal);
7556 Ops.push_back(Chain);
7557
7558 // We need to enforce the calling convention for the callsite, so that
7559 // argument ordering is enforced correctly, and that register allocation can
7560 // see that some registers may be assumed clobbered and have to preserve
7561 // them across calls to the intrinsic.
7562 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7563 sdl, NodeTys, Ops);
7564 SDValue patchableNode = SDValue(MN, 0);
7565 DAG.setRoot(patchableNode);
7566 setValue(&I, patchableNode);
7567 return;
7568 }
7569 case Intrinsic::xray_typedevent: {
7570 // Here we want to make sure that the intrinsic behaves as if it has a
7571 // specific calling convention.
7572 const auto &Triple = DAG.getTarget().getTargetTriple();
7573 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7574 return;
7575
7577
7578 // We want to say that we always want the arguments in registers.
7579 // It's unclear to me how manipulating the selection DAG here forces callers
7580 // to provide arguments in registers instead of on the stack.
7581 SDValue LogTypeId = getValue(I.getArgOperand(0));
7582 SDValue LogEntryVal = getValue(I.getArgOperand(1));
7583 SDValue StrSizeVal = getValue(I.getArgOperand(2));
7584 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7585 SDValue Chain = getRoot();
7586 Ops.push_back(LogTypeId);
7587 Ops.push_back(LogEntryVal);
7588 Ops.push_back(StrSizeVal);
7589 Ops.push_back(Chain);
7590
7591 // We need to enforce the calling convention for the callsite, so that
7592 // argument ordering is enforced correctly, and that register allocation can
7593 // see that some registers may be assumed clobbered and have to preserve
7594 // them across calls to the intrinsic.
7596 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7597 SDValue patchableNode = SDValue(MN, 0);
7598 DAG.setRoot(patchableNode);
7599 setValue(&I, patchableNode);
7600 return;
7601 }
7602 case Intrinsic::experimental_deoptimize:
7604 return;
7605 case Intrinsic::experimental_stepvector:
7606 visitStepVector(I);
7607 return;
7608 case Intrinsic::vector_reduce_fadd:
7609 case Intrinsic::vector_reduce_fmul:
7610 case Intrinsic::vector_reduce_add:
7611 case Intrinsic::vector_reduce_mul:
7612 case Intrinsic::vector_reduce_and:
7613 case Intrinsic::vector_reduce_or:
7614 case Intrinsic::vector_reduce_xor:
7615 case Intrinsic::vector_reduce_smax:
7616 case Intrinsic::vector_reduce_smin:
7617 case Intrinsic::vector_reduce_umax:
7618 case Intrinsic::vector_reduce_umin:
7619 case Intrinsic::vector_reduce_fmax:
7620 case Intrinsic::vector_reduce_fmin:
7621 case Intrinsic::vector_reduce_fmaximum:
7622 case Intrinsic::vector_reduce_fminimum:
7623 visitVectorReduce(I, Intrinsic);
7624 return;
7625
7626 case Intrinsic::icall_branch_funnel: {
7628 Ops.push_back(getValue(I.getArgOperand(0)));
7629
7630 int64_t Offset;
7631 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7632 I.getArgOperand(1), Offset, DAG.getDataLayout()));
7633 if (!Base)
7635 "llvm.icall.branch.funnel operand must be a GlobalValue");
7636 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7637
7638 struct BranchFunnelTarget {
7639 int64_t Offset;
7641 };
7643
7644 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7645 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7646 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7647 if (ElemBase != Base)
7648 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7649 "to the same GlobalValue");
7650
7651 SDValue Val = getValue(I.getArgOperand(Op + 1));
7652 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7653 if (!GA)
7655 "llvm.icall.branch.funnel operand must be a GlobalValue");
7657 GA->getGlobal(), sdl, Val.getValueType(),
7658 GA->getOffset())});
7659 }
7660 llvm::sort(Targets,
7661 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7662 return T1.Offset < T2.Offset;
7663 });
7664
7665 for (auto &T : Targets) {
7666 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7667 Ops.push_back(T.Target);
7668 }
7669
7670 Ops.push_back(DAG.getRoot()); // Chain
7671 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7672 MVT::Other, Ops),
7673 0);
7674 DAG.setRoot(N);
7675 setValue(&I, N);
7676 HasTailCall = true;
7677 return;
7678 }
7679
7680 case Intrinsic::wasm_landingpad_index:
7681 // Information this intrinsic contained has been transferred to
7682 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7683 // delete it now.
7684 return;
7685
7686 case Intrinsic::aarch64_settag:
7687 case Intrinsic::aarch64_settag_zero: {
7689 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7691 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7692 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7693 ZeroMemory);
7694 DAG.setRoot(Val);
7695 setValue(&I, Val);
7696 return;
7697 }
7698 case Intrinsic::amdgcn_cs_chain: {
7699 assert(I.arg_size() == 5 && "Additional args not supported yet");
7700 assert(cast<ConstantInt>(I.getOperand(4))->isZero() &&
7701 "Non-zero flags not supported yet");
7702
7703 // At this point we don't care if it's amdgpu_cs_chain or
7704 // amdgpu_cs_chain_preserve.
7706
7707 Type *RetTy = I.getType();
7708 assert(RetTy->isVoidTy() && "Should not return");
7709
7710 SDValue Callee = getValue(I.getOperand(0));
7711
7712 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
7713 // We'll also tack the value of the EXEC mask at the end.
7715 Args.reserve(3);
7716
7717 for (unsigned Idx : {2, 3, 1}) {
7719 Arg.Node = getValue(I.getOperand(Idx));
7720 Arg.Ty = I.getOperand(Idx)->getType();
7721 Arg.setAttributes(&I, Idx);
7722 Args.push_back(Arg);
7723 }
7724
7725 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
7726 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
7727 Args[2].IsInReg = true; // EXEC should be inreg
7728
7730 CLI.setDebugLoc(getCurSDLoc())
7731 .setChain(getRoot())
7732 .setCallee(CC, RetTy, Callee, std::move(Args))
7733 .setNoReturn(true)
7734 .setTailCall(true)
7735 .setConvergent(I.isConvergent());
7736 CLI.CB = &I;
7737 std::pair<SDValue, SDValue> Result =
7738 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
7739 (void)Result;
7740 assert(!Result.first.getNode() && !Result.second.getNode() &&
7741 "Should've lowered as tail call");
7742
7743 HasTailCall = true;
7744 return;
7745 }
7746 case Intrinsic::ptrmask: {
7747 SDValue Ptr = getValue(I.getOperand(0));
7748 SDValue Mask = getValue(I.getOperand(1));
7749
7750 EVT PtrVT = Ptr.getValueType();
7751 assert(PtrVT == Mask.getValueType() &&
7752 "Pointers with different index type are not supported by SDAG");
7753 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
7754 return;
7755 }
7756 case Intrinsic::threadlocal_address: {
7757 setValue(&I, getValue(I.getOperand(0)));
7758 return;
7759 }
7760 case Intrinsic::get_active_lane_mask: {
7761 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7762 SDValue Index = getValue(I.getOperand(0));
7763 EVT ElementVT = Index.getValueType();
7764
7765 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
7766 visitTargetIntrinsic(I, Intrinsic);
7767 return;
7768 }
7769
7770 SDValue TripCount = getValue(I.getOperand(1));
7771 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
7772 CCVT.getVectorElementCount());
7773
7774 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
7775 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
7776 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
7777 SDValue VectorInduction = DAG.getNode(
7778 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
7779 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
7780 VectorTripCount, ISD::CondCode::SETULT);
7781 setValue(&I, SetCC);
7782 return;
7783 }
7784 case Intrinsic::experimental_get_vector_length: {
7785 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
7786 "Expected positive VF");
7787 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
7788 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
7789
7790 SDValue Count = getValue(I.getOperand(0));
7791 EVT CountVT = Count.getValueType();
7792
7793 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
7794 visitTargetIntrinsic(I, Intrinsic);
7795 return;
7796 }
7797
7798 // Expand to a umin between the trip count and the maximum elements the type
7799 // can hold.
7800 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7801
7802 // Extend the trip count to at least the result VT.
7803 if (CountVT.bitsLT(VT)) {
7804 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
7805 CountVT = VT;
7806 }
7807
7808 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
7809 ElementCount::get(VF, IsScalable));
7810
7811 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
7812 // Clip to the result type if needed.
7813 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
7814
7815 setValue(&I, Trunc);
7816 return;
7817 }
7818 case Intrinsic::experimental_cttz_elts: {
7819 auto DL = getCurSDLoc();
7820 SDValue Op = getValue(I.getOperand(0));
7821 EVT OpVT = Op.getValueType();
7822
7823 if (!TLI.shouldExpandCttzElements(OpVT)) {
7824 visitTargetIntrinsic(I, Intrinsic);
7825 return;
7826 }
7827
7828 if (OpVT.getScalarType() != MVT::i1) {
7829 // Compare the input vector elements to zero & use to count trailing zeros
7830 SDValue AllZero = DAG.getConstant(0, DL, OpVT);
7831 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
7832 OpVT.getVectorElementCount());
7833 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
7834 }
7835
7836 // Find the smallest "sensible" element type to use for the expansion.
7837 ConstantRange CR(
7839 if (OpVT.isScalableVT())
7840 CR = CR.umul_sat(getVScaleRange(I.getCaller(), 64));
7841
7842 // If the zero-is-poison flag is set, we can assume the upper limit
7843 // of the result is VF-1.
7844 if (!cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero())
7845 CR = CR.subtract(APInt(64, 1));
7846
7847 unsigned EltWidth = I.getType()->getScalarSizeInBits();
7848 EltWidth = std::min(EltWidth, (unsigned)CR.getActiveBits());
7849 EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);
7850
7851 MVT NewEltTy = MVT::getIntegerVT(EltWidth);
7852
7853 // Create the new vector type & get the vector length
7854 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,
7855 OpVT.getVectorElementCount());
7856
7857 SDValue VL =
7858 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());
7859
7860 SDValue StepVec = DAG.getStepVector(DL, NewVT);
7861 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
7862 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
7864 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
7866 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
7867
7868 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7870
7871 setValue(&I, Ret);
7872 return;
7873 }
7874 case Intrinsic::vector_insert: {
7875 SDValue Vec = getValue(I.getOperand(0));
7876 SDValue SubVec = getValue(I.getOperand(1));
7877 SDValue Index = getValue(I.getOperand(2));
7878
7879 // The intrinsic's index type is i64, but the SDNode requires an index type
7880 // suitable for the target. Convert the index as required.
7881 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7882 if (Index.getValueType() != VectorIdxTy)
7883 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
7884
7885 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7886 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
7887 Index));
7888 return;
7889 }
7890 case Intrinsic::vector_extract: {
7891 SDValue Vec = getValue(I.getOperand(0));
7892 SDValue Index = getValue(I.getOperand(1));
7893 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7894
7895 // The intrinsic's index type is i64, but the SDNode requires an index type
7896 // suitable for the target. Convert the index as required.
7897 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
7898 if (Index.getValueType() != VectorIdxTy)
7899 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
7900
7901 setValue(&I,
7902 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
7903 return;
7904 }
7905 case Intrinsic::experimental_vector_reverse:
7906 visitVectorReverse(I);
7907 return;
7908 case Intrinsic::experimental_vector_splice:
7909 visitVectorSplice(I);
7910 return;
7911 case Intrinsic::callbr_landingpad:
7912 visitCallBrLandingPad(I);
7913 return;
7914 case Intrinsic::experimental_vector_interleave2:
7915 visitVectorInterleave(I);
7916 return;
7917 case Intrinsic::experimental_vector_deinterleave2:
7918 visitVectorDeinterleave(I);
7919 return;
7920 case Intrinsic::experimental_convergence_anchor:
7921 case Intrinsic::experimental_convergence_entry:
7922 case Intrinsic::experimental_convergence_loop:
7923 visitConvergenceControl(I, Intrinsic);
7924 }
7925}
7926
7927void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
7928 const ConstrainedFPIntrinsic &FPI) {
7929 SDLoc sdl = getCurSDLoc();
7930
7931 // We do not need to serialize constrained FP intrinsics against
7932 // each other or against (nonvolatile) loads, so they can be
7933 // chained like loads.
7934 SDValue Chain = DAG.getRoot();
7936 Opers.push_back(Chain);
7937 if (FPI.isUnaryOp()) {
7938 Opers.push_back(getValue(FPI.getArgOperand(0)));
7939 } else if (FPI.isTernaryOp()) {
7940 Opers.push_back(getValue(FPI.getArgOperand(0)));
7941 Opers.push_back(getValue(FPI.getArgOperand(1)));
7942 Opers.push_back(getValue(FPI.getArgOperand(2)));
7943 } else {
7944 Opers.push_back(getValue(FPI.getArgOperand(0)));
7945 Opers.push_back(getValue(FPI.getArgOperand(1)));
7946 }
7947
7948 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
7949 assert(Result.getNode()->getNumValues() == 2);
7950
7951 // Push node to the appropriate list so that future instructions can be
7952 // chained up correctly.
7953 SDValue OutChain = Result.getValue(1);
7954 switch (EB) {
7956 // The only reason why ebIgnore nodes still need to be chained is that
7957 // they might depend on the current rounding mode, and therefore must
7958 // not be moved across instruction that may change that mode.
7959 [[fallthrough]];
7961 // These must not be moved across calls or instructions that may change
7962 // floating-point exception masks.
7963 PendingConstrainedFP.push_back(OutChain);
7964 break;
7966 // These must not be moved across calls or instructions that may change
7967 // floating-point exception masks or read floating-point exception flags.
7968 // In addition, they cannot be optimized out even if unused.
7969 PendingConstrainedFPStrict.push_back(OutChain);
7970 break;
7971 }
7972 };
7973
7975 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
7976 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
7978
7981 Flags.setNoFPExcept(true);
7982
7983 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
7984 Flags.copyFMF(*FPOp);
7985
7986 unsigned Opcode;
7987 switch (FPI.getIntrinsicID()) {
7988 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7989#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
7990 case Intrinsic::INTRINSIC: \
7991 Opcode = ISD::STRICT_##DAGN; \
7992 break;
7993#include "llvm/IR/ConstrainedOps.def"
7994 case Intrinsic::experimental_constrained_fmuladd: {
7995 Opcode = ISD::STRICT_FMA;
7996 // Break fmuladd into fmul and fadd.
7999 Opers.pop_back();
8000 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8001 pushOutChain(Mul, EB);
8002 Opcode = ISD::STRICT_FADD;
8003 Opers.clear();
8004 Opers.push_back(Mul.getValue(1));
8005 Opers.push_back(Mul.getValue(0));
8006 Opers.push_back(getValue(FPI.getArgOperand(2)));
8007 }
8008 break;
8009 }
8010 }
8011
8012 // A few strict DAG nodes carry additional operands that are not
8013 // set up by the default code above.
8014 switch (Opcode) {
8015 default: break;
8017 Opers.push_back(
8019 break;
8020 case ISD::STRICT_FSETCC:
8021 case ISD::STRICT_FSETCCS: {
8022 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8023 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8024 if (TM.Options.NoNaNsFPMath)
8025 Condition = getFCmpCodeWithoutNaN(Condition);
8026 Opers.push_back(DAG.getCondCode(Condition));
8027 break;
8028 }
8029 }
8030
8031 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8032 pushOutChain(Result, EB);
8033
8034 SDValue FPResult = Result.getValue(0);
8035 setValue(&FPI, FPResult);
8036}
8037
8038static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8039 std::optional<unsigned> ResOPC;
8040 switch (VPIntrin.getIntrinsicID()) {
8041 case Intrinsic::vp_ctlz: {
8042 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8043 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8044 break;
8045 }
8046 case Intrinsic::vp_cttz: {
8047 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8048 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8049 break;
8050 }
8051#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8052 case Intrinsic::VPID: \
8053 ResOPC = ISD::VPSD; \
8054 break;
8055#include "llvm/IR/VPIntrinsics.def"
8056 }
8057
8058 if (!ResOPC)
8060 "Inconsistency: no SDNode available for this VPIntrinsic!");
8061
8062 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8063 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8064 if (VPIntrin.getFastMathFlags().allowReassoc())
8065 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8066 : ISD::VP_REDUCE_FMUL;
8067 }
8068
8069 return *ResOPC;
8070}
8071
8072void SelectionDAGBuilder::visitVPLoad(
8073 const VPIntrinsic &VPIntrin, EVT VT,
8074 const SmallVectorImpl<SDValue> &OpValues) {
8075 SDLoc DL = getCurSDLoc();
8076 Value *PtrOperand = VPIntrin.getArgOperand(0);
8077 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8078 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8079 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8080 SDValue LD;
8081 // Do not serialize variable-length loads of constant memory with
8082 // anything.
8083 if (!Alignment)
8084 Alignment = DAG.getEVTAlign(VT);
8085 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8086 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8087 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8090 MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
8091 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
8092 MMO, false /*IsExpanding */);
8093 if (AddToChain)
8094 PendingLoads.push_back(LD.getValue(1));
8095 setValue(&VPIntrin, LD);
8096}
8097
8098void SelectionDAGBuilder::visitVPGather(
8099 const VPIntrinsic &VPIntrin, EVT VT,
8100 const SmallVectorImpl<SDValue> &OpValues) {
8101 SDLoc DL = getCurSDLoc();
8103 Value *PtrOperand = VPIntrin.getArgOperand(0);
8104 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8105 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8106 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8107 SDValue LD;
8108 if (!Alignment)
8109 Alignment = DAG.getEVTAlign(VT.getScalarType());
8110 unsigned AS =
8111 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8114 MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
8115 SDValue Base, Index, Scale;
8116 ISD::MemIndexType IndexType;
8117 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8118 this, VPIntrin.getParent(),
8119 VT.getScalarStoreSize());
8120 if (!UniformBase) {
8122 Index = getValue(PtrOperand);
8123 IndexType = ISD::SIGNED_SCALED;
8125 }
8126 EVT IdxVT = Index.getValueType();
8127 EVT EltTy = IdxVT.getVectorElementType();
8128 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8129 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8130 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8131 }
8132 LD = DAG.getGatherVP(
8133 DAG.getVTList(VT, MVT::Other), VT, DL,
8134 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8135 IndexType);
8136 PendingLoads.push_back(LD.getValue(1));
8137 setValue(&VPIntrin, LD);
8138}
8139
8140void SelectionDAGBuilder::visitVPStore(
8141 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8142 SDLoc DL = getCurSDLoc();
8143 Value *PtrOperand = VPIntrin.getArgOperand(1);
8144 EVT VT = OpValues[0].getValueType();
8145 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8146 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8147 SDValue ST;
8148 if (!Alignment)
8149 Alignment = DAG.getEVTAlign(VT);
8150 SDValue Ptr = OpValues[1];
8151 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
8154 MemoryLocation::UnknownSize, *Alignment, AAInfo);
8155 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
8156 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
8157 /* IsTruncating */ false, /*IsCompressing*/ false);
8158 DAG.setRoot(ST);
8159 setValue(&VPIntrin, ST);
8160}
8161
8162void SelectionDAGBuilder::visitVPScatter(
8163 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8164 SDLoc DL = getCurSDLoc();
8166 Value *PtrOperand = VPIntrin.getArgOperand(1);
8167 EVT VT = OpValues[0].getValueType();
8168 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8169 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8170 SDValue ST;
8171 if (!Alignment)
8172 Alignment = DAG.getEVTAlign(VT.getScalarType());
8173 unsigned AS =
8174 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8177 MemoryLocation::UnknownSize, *Alignment, AAInfo);
8178 SDValue Base, Index, Scale;
8179 ISD::MemIndexType IndexType;
8180 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8181 this, VPIntrin.getParent(),
8182 VT.getScalarStoreSize());
8183 if (!UniformBase) {
8185 Index = getValue(PtrOperand);
8186 IndexType = ISD::SIGNED_SCALED;
8187 Scale =
8189 }
8190 EVT IdxVT = Index.getValueType();
8191 EVT EltTy = IdxVT.getVectorElementType();
8192 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8193 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8194 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8195 }
8196 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
8197 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8198 OpValues[2], OpValues[3]},
8199 MMO, IndexType);
8200 DAG.setRoot(ST);
8201 setValue(&VPIntrin, ST);
8202}
8203
8204void SelectionDAGBuilder::visitVPStridedLoad(
8205 const VPIntrinsic &VPIntrin, EVT VT,
8206 const SmallVectorImpl<SDValue> &OpValues) {
8207 SDLoc DL = getCurSDLoc();
8208 Value *PtrOperand = VPIntrin.getArgOperand(0);
8209 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8210 if (!Alignment)
8211 Alignment = DAG.getEVTAlign(VT.getScalarType());
8212 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8213 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8214 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8215 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8216 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8217 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8220 MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
8221
8222 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
8223 OpValues[2], OpValues[3], MMO,
8224 false /*IsExpanding*/);
8225
8226 if (AddToChain)
8227 PendingLoads.push_back(LD.getValue(1));
8228 setValue(&VPIntrin, LD);
8229}
8230
8231void SelectionDAGBuilder::visitVPStridedStore(
8232 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8233 SDLoc DL = getCurSDLoc();
8234 Value *PtrOperand = VPIntrin.getArgOperand(1);
8235 EVT VT = OpValues[0].getValueType();
8236 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8237 if (!Alignment)
8238 Alignment = DAG.getEVTAlign(VT.getScalarType());
8239 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8240 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8243 MemoryLocation::UnknownSize, *Alignment, AAInfo);
8244
8246 getMemoryRoot(), DL, OpValues[0], OpValues[1],
8247 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
8248 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
8249 /*IsCompressing*/ false);
8250
8251 DAG.setRoot(ST);
8252 setValue(&VPIntrin, ST);
8253}
8254
8255void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8257 SDLoc DL = getCurSDLoc();
8258
8259 ISD::CondCode Condition;
8261 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
8262 if (IsFP) {
8263 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
8264 // flags, but calls that don't return floating-point types can't be
8265 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
8266 Condition = getFCmpCondCode(CondCode);
8267 if (TM.Options.NoNaNsFPMath)
8268 Condition = getFCmpCodeWithoutNaN(Condition);
8269 } else {
8270 Condition = getICmpCondCode(CondCode);
8271 }
8272
8273 SDValue Op1 = getValue(VPIntrin.getOperand(0));
8274 SDValue Op2 = getValue(VPIntrin.getOperand(1));
8275 // #2 is the condition code
8276 SDValue MaskOp = getValue(VPIntrin.getOperand(3));
8277 SDValue EVL = getValue(VPIntrin.getOperand(4));
8278 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8279 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8280 "Unexpected target EVL type");
8281 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
8282
8284 VPIntrin.getType());
8285 setValue(&VPIntrin,
8286 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
8287}
8288
8289void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8290 const VPIntrinsic &VPIntrin) {
8291 SDLoc DL = getCurSDLoc();
8292 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8293
8294 auto IID = VPIntrin.getIntrinsicID();
8295
8296 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
8297 return visitVPCmp(*CmpI);
8298
8299 SmallVector<EVT, 4> ValueVTs;
8301 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
8302 SDVTList VTs = DAG.getVTList(ValueVTs);
8303
8304 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
8305
8306 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8307 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8308 "Unexpected target EVL type");
8309
8310 // Request operands.
8311 SmallVector<SDValue, 7> OpValues;
8312 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8313 auto Op = getValue(VPIntrin.getArgOperand(I));
8314 if (I == EVLParamPos)
8315 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
8316 OpValues.push_back(Op);
8317 }
8318
8319 switch (Opcode) {
8320 default: {
8321 SDNodeFlags SDFlags;
8322 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8323 SDFlags.copyFMF(*FPMO);
8324 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
8325 setValue(&VPIntrin, Result);
8326 break;
8327 }
8328 case ISD::VP_LOAD:
8329 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
8330 break;
8331 case ISD::VP_GATHER:
8332 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
8333 break;
8334 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8335 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
8336 break;
8337 case ISD::VP_STORE:
8338 visitVPStore(VPIntrin, OpValues);
8339 break;
8340 case ISD::VP_SCATTER:
8341 visitVPScatter(VPIntrin, OpValues);
8342 break;
8343 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8344 visitVPStridedStore(VPIntrin, OpValues);
8345 break;
8346 case ISD::VP_FMULADD: {
8347 assert(OpValues.size() == 5 && "Unexpected number of operands");
8348 SDNodeFlags SDFlags;
8349 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8350 SDFlags.copyFMF(*FPMO);
8353 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
8354 } else {
8356 ISD::VP_FMUL, DL, VTs,
8357 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
8358 SDValue Add =
8359 DAG.getNode(ISD::VP_FADD, DL, VTs,
8360 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
8361 setValue(&VPIntrin, Add);
8362 }
8363 break;
8364 }
8365 case ISD::VP_IS_FPCLASS: {
8366 const DataLayout DLayout = DAG.getDataLayout();
8367 EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType());
8368 auto Constant = OpValues[1]->getAsZExtVal();
8370 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT,
8371 {OpValues[0], Check, OpValues[2], OpValues[3]});
8372 setValue(&VPIntrin, V);
8373 return;
8374 }
8375 case ISD::VP_INTTOPTR: {
8376 SDValue N = OpValues[0];
8377 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
8378 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
8379 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8380 OpValues[2]);
8381 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8382 OpValues[2]);
8383 setValue(&VPIntrin, N);
8384 break;
8385 }
8386 case ISD::VP_PTRTOINT: {
8387 SDValue N = OpValues[0];
8389 VPIntrin.getType());
8390 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
8391 VPIntrin.getOperand(0)->getType());
8392 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8393 OpValues[2]);
8394 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8395 OpValues[2]);
8396 setValue(&VPIntrin, N);
8397 break;
8398 }
8399 case ISD::VP_ABS:
8400 case ISD::VP_CTLZ:
8401 case ISD::VP_CTLZ_ZERO_UNDEF:
8402 case ISD::VP_CTTZ:
8403 case ISD::VP_CTTZ_ZERO_UNDEF: {
8404 SDValue Result =
8405 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
8406 setValue(&VPIntrin, Result);
8407 break;
8408 }
8409 }
8410}
8411
8412SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
8413 const BasicBlock *EHPadBB,
8414 MCSymbol *&BeginLabel) {
8416 MachineModuleInfo &MMI = MF.getMMI();
8417
8418 // Insert a label before the invoke call to mark the try range. This can be
8419 // used to detect deletion of the invoke via the MachineModuleInfo.
8420 BeginLabel = MMI.getContext().createTempSymbol();
8421
8422 // For SjLj, keep track of which landing pads go with which invokes
8423 // so as to maintain the ordering of pads in the LSDA.
8424 unsigned CallSiteIndex = MMI.getCurrentCallSite();
8425 if (CallSiteIndex) {
8426 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
8427 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
8428
8429 // Now that the call site is handled, stop tracking it.
8430 MMI.setCurrentCallSite(0);
8431 }
8432
8433 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
8434}
8435
8436SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
8437 const BasicBlock *EHPadBB,
8438 MCSymbol *BeginLabel) {
8439 assert(BeginLabel && "BeginLabel should've been set");
8440
8442 MachineModuleInfo &MMI = MF.getMMI();
8443
8444 // Insert a label at the end of the invoke call to mark the try range. This
8445 // can be used to detect deletion of the invoke via the MachineModuleInfo.
8446 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
8447 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
8448
8449 // Inform MachineModuleInfo of range.
8451 // There is a platform (e.g. wasm) that uses funclet style IR but does not
8452 // actually use outlined funclets and their LSDA info style.
8453 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
8454 assert(II && "II should've been set");
8455 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
8456 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
8457 } else if (!isScopedEHPersonality(Pers)) {
8458 assert(EHPadBB);
8459 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
8460 }
8461
8462 return Chain;
8463}
8464
8465std::pair<SDValue, SDValue>
8467 const BasicBlock *EHPadBB) {
8468 MCSymbol *BeginLabel = nullptr;
8469
8470 if (EHPadBB) {
8471 // Both PendingLoads and PendingExports must be flushed here;
8472 // this call might not return.
8473 (void)getRoot();
8474 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
8475 CLI.setChain(getRoot());
8476 }
8477
8479 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8480
8481 assert((CLI.IsTailCall || Result.second.getNode()) &&
8482 "Non-null chain expected with non-tail call!");
8483 assert((Result.second.getNode() || !Result.first.getNode()) &&
8484 "Null value expected with tail call!");
8485
8486 if (!Result.second.getNode()) {
8487 // As a special case, a null chain means that a tail call has been emitted
8488 // and the DAG root is already updated.
8489 HasTailCall = true;
8490
8491 // Since there's no actual continuation from this block, nothing can be
8492 // relying on us setting vregs for them.
8493 PendingExports.clear();
8494 } else {
8495 DAG.setRoot(Result.second);
8496 }
8497
8498 if (EHPadBB) {
8499 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
8500 BeginLabel));
8501 }
8502
8503 return Result;
8504}
8505
8507 bool isTailCall,
8508 bool isMustTailCall,
8509 const BasicBlock *EHPadBB) {
8510 auto &DL = DAG.getDataLayout();
8511 FunctionType *FTy = CB.getFunctionType();
8512 Type *RetTy = CB.getType();
8513
8515 Args.reserve(CB.arg_size());
8516
8517 const Value *SwiftErrorVal = nullptr;
8519
8520 if (isTailCall) {
8521 // Avoid emitting tail calls in functions with the disable-tail-calls
8522 // attribute.
8523 auto *Caller = CB.getParent()->getParent();
8524 if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
8525 "true" && !isMustTailCall)
8526 isTailCall = false;
8527
8528 // We can't tail call inside a function with a swifterror argument. Lowering
8529 // does not support this yet. It would have to move into the swifterror
8530 // register before the call.
8531 if (TLI.supportSwiftError() &&
8532 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8533 isTailCall = false;
8534 }
8535
8536 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
8538 const Value *V = *I;
8539
8540 // Skip empty types
8541 if (V->getType()->isEmptyTy())
8542 continue;
8543
8544 SDValue ArgNode = getValue(V);
8545 Entry.Node = ArgNode; Entry.Ty = V->getType();
8546
8547 Entry.setAttributes(&CB, I - CB.arg_begin());
8548
8549 // Use swifterror virtual register as input to the call.
8550 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
8551 SwiftErrorVal = V;
8552 // We find the virtual register for the actual swifterror argument.
8553 // Instead of using the Value, we use the virtual register instead.
8554 Entry.Node =
8556 EVT(TLI.getPointerTy(DL)));
8557 }
8558
8559 Args.push_back(Entry);
8560
8561 // If we have an explicit sret argument that is an Instruction, (i.e., it
8562 // might point to function-local memory), we can't meaningfully tail-call.
8563 if (Entry.IsSRet && isa<Instruction>(V))
8564 isTailCall = false;
8565 }
8566
8567 // If call site has a cfguardtarget operand bundle, create and add an
8568 // additional ArgListEntry.
8569 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
8571 Value *V = Bundle->Inputs[0];
8572 SDValue ArgNode = getValue(V);
8573 Entry.Node = ArgNode;
8574 Entry.Ty = V->getType();
8575 Entry.IsCFGuardTarget = true;
8576 Args.push_back(Entry);
8577 }
8578
8579 // Check if target-independent constraints permit a tail call here.
8580 // Target-dependent constraints are checked within TLI->LowerCallTo.
8581 if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
8582 isTailCall = false;
8583
8584 // Disable tail calls if there is an swifterror argument. Targets have not
8585 // been updated to support tail calls.
8586 if (TLI.supportSwiftError() && SwiftErrorVal)
8587 isTailCall = false;
8588
8589 ConstantInt *CFIType = nullptr;
8590 if (CB.isIndirectCall()) {
8591 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
8592 if (!TLI.supportKCFIBundles())
8594 "Target doesn't support calls with kcfi operand bundles.");
8595 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8596 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8597 }
8598 }
8599
8600 SDValue ConvControlToken;
8601 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8602 auto *Token = Bundle->Inputs[0].get();
8603 ConvControlToken = getValue(Token);
8604 } else {
8605 ConvControlToken = DAG.getUNDEF(MVT::Untyped);
8606 }
8607
8610 .setChain(getRoot())
8611 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
8612 .setTailCall(isTailCall)
8616 .setCFIType(CFIType)
8617 .setConvergenceControlToken(ConvControlToken);
8618 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8619
8620 if (Result.first.getNode()) {
8621 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
8622 setValue(&CB, Result.first);
8623 }
8624
8625 // The last element of CLI.InVals has the SDValue for swifterror return.
8626 // Here we copy it to a virtual register and update SwiftErrorMap for
8627 // book-keeping.
8628 if (SwiftErrorVal && TLI.supportSwiftError()) {
8629 // Get the last element of InVals.
8630 SDValue Src = CLI.InVals.back();
8631 Register VReg =
8632 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
8633 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
8634 DAG.setRoot(CopyNode);
8635 }
8636}
8637
8638static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
8639 SelectionDAGBuilder &Builder) {
8640 // Check to see if this load can be trivially constant folded, e.g. if the
8641 // input is from a string literal.
8642 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
8643 // Cast pointer to the type we really want to load.
8644 Type *LoadTy =
8645 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
8646 if (LoadVT.isVector())
8647 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
8648
8649 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
8650 PointerType::getUnqual(LoadTy));
8651
8652 if (const Constant *LoadCst =
8653 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
8654 LoadTy, Builder.DAG.getDataLayout()))
8655 return Builder.getValue(LoadCst);
8656 }
8657
8658 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
8659 // still constant memory, the input chain can be the entry node.
8660 SDValue Root;
8661 bool ConstantMemory = false;
8662
8663 // Do not serialize (non-volatile) loads of constant memory with anything.
8664 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
8665 Root = Builder.DAG.getEntryNode();
8666 ConstantMemory = true;
8667 } else {
8668 // Do not serialize non-volatile loads against each other.
8669 Root = Builder.DAG.getRoot();
8670 }
8671
8672 SDValue Ptr = Builder.getValue(PtrVal);
8673 SDValue LoadVal =
8674 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
8675 MachinePointerInfo(PtrVal), Align(1));
8676
8677 if (!ConstantMemory)
8678 Builder.PendingLoads.push_back(LoadVal.getValue(1));
8679 return LoadVal;
8680}
8681
8682/// Record the value for an instruction that produces an integer result,
8683/// converting the type where necessary.
8684void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
8685 SDValue Value,
8686 bool IsSigned) {
8688 I.getType(), true);
8689 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
8690 setValue(&I, Value);
8691}
8692
8693/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
8694/// true and lower it. Otherwise return false, and it will be lowered like a
8695/// normal call.
8696/// The caller already checked that \p I calls the appropriate LibFunc with a
8697/// correct prototype.
8698bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
8699 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
8700 const Value *Size = I.getArgOperand(2);
8701 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
8702 if (CSize && CSize->getZExtValue() == 0) {
8704 I.getType(), true);
8705 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
8706 return true;
8707 }
8708
8710 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
8711 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
8713 if (Res.first.getNode()) {
8714 processIntegerCallValue(I, Res.first, true);
8715 PendingLoads.push_back(Res.second);
8716 return true;
8717 }
8718
8719 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
8720 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
8721 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
8722 return false;
8723
8724 // If the target has a fast compare for the given size, it will return a
8725 // preferred load type for that size. Require that the load VT is legal and
8726 // that the target supports unaligned loads of that type. Otherwise, return
8727 // INVALID.
8728 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
8730 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
8731 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
8732 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
8733 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
8734 // TODO: Check alignment of src and dest ptrs.
8735 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
8736 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
8737 if (!TLI.isTypeLegal(LVT) ||
8738 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
8739 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
8741 }
8742
8743 return LVT;
8744 };
8745
8746 // This turns into unaligned loads. We only do this if the target natively
8747 // supports the MVT we'll be loading or if it is small enough (<= 4) that
8748 // we'll only produce a small number of byte loads.
8749 MVT LoadVT;
8750 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
8751 switch (NumBitsToCompare) {
8752 default:
8753 return false;
8754 case 16:
8755 LoadVT = MVT::i16;
8756 break;
8757 case 32:
8758 LoadVT = MVT::i32;
8759 break;
8760 case 64:
8761 case 128:
8762 case 256:
8763 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
8764 break;
8765 }
8766
8767 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
8768 return false;
8769
8770 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
8771 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
8772
8773 // Bitcast to a wide integer type if the loads are vectors.
8774 if (LoadVT.isVector()) {
8775 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
8776 LoadL = DAG.getBitcast(CmpVT, LoadL);
8777 LoadR = DAG.getBitcast(CmpVT, LoadR);
8778 }
8779
8780 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
8781 processIntegerCallValue(I, Cmp, false);
8782 return true;
8783}
8784
8785/// See if we can lower a memchr call into an optimized form. If so, return
8786/// true and lower it. Otherwise return false, and it will be lowered like a
8787/// normal call.
8788/// The caller already checked that \p I calls the appropriate LibFunc with a
8789/// correct prototype.
8790bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
8791 const Value *Src = I.getArgOperand(0);
8792 const Value *Char = I.getArgOperand(1);
8793 const Value *Length = I.getArgOperand(2);
8794
8796 std::pair<SDValue, SDValue> Res =
8798 getValue(Src), getValue(Char), getValue(Length),
8799 MachinePointerInfo(Src));
8800 if (Res.first.getNode()) {
8801 setValue(&I, Res.first);
8802 PendingLoads.push_back(Res.second);
8803 return true;
8804 }
8805
8806 return false;
8807}
8808
8809/// See if we can lower a mempcpy call into an optimized form. If so, return
8810/// true and lower it. Otherwise return false, and it will be lowered like a
8811/// normal call.
8812/// The caller already checked that \p I calls the appropriate LibFunc with a
8813/// correct prototype.
8814bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
8815 SDValue Dst = getValue(I.getArgOperand(0));
8816 SDValue Src = getValue(I.getArgOperand(1));
8817 SDValue Size = getValue(I.getArgOperand(2));
8818
8819 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
8820 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
8821 // DAG::getMemcpy needs Alignment to be defined.
8822 Align Alignment = std::min(DstAlign, SrcAlign);
8823
8824 SDLoc sdl = getCurSDLoc();
8825
8826 // In the mempcpy context we need to pass in a false value for isTailCall
8827 // because the return pointer needs to be adjusted by the size of
8828 // the copied memory.
8829 SDValue Root = getMemoryRoot();
8830 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, false, false,
8831 /*isTailCall=*/false,
8832 MachinePointerInfo(I.getArgOperand(0)),
8833 MachinePointerInfo(I.getArgOperand(1)),
8834 I.getAAMetadata());
8835 assert(MC.getNode() != nullptr &&
8836 "** memcpy should not be lowered as TailCall in mempcpy context **");
8837 DAG.setRoot(MC);
8838
8839 // Check if Size needs to be truncated or extended.
8840 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
8841
8842 // Adjust return pointer to point just past the last dst byte.
8843 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
8844 Dst, Size);
8845 setValue(&I, DstPlusSize);
8846 return true;
8847}
8848
8849/// See if we can lower a strcpy call into an optimized form. If so, return
8850/// true and lower it, otherwise return false and it will be lowered like a
8851/// normal call.
8852/// The caller already checked that \p I calls the appropriate LibFunc with a
8853/// correct prototype.
8854bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
8855 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8856
8858 std::pair<SDValue, SDValue> Res =
8860 getValue(Arg0), getValue(Arg1),
8861 MachinePointerInfo(Arg0),
8862 MachinePointerInfo(Arg1), isStpcpy);
8863 if (Res.first.getNode()) {
8864 setValue(&I, Res.first);
8865 DAG.setRoot(Res.second);
8866 return true;
8867 }
8868
8869 return false;
8870}
8871
8872/// See if we can lower a strcmp call into an optimized form. If so, return
8873/// true and lower it, otherwise return false and it will be lowered like a
8874/// normal call.
8875/// The caller already checked that \p I calls the appropriate LibFunc with a
8876/// correct prototype.
8877bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
8878 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8879
8881 std::pair<SDValue, SDValue> Res =
8883 getValue(Arg0), getValue(Arg1),
8884 MachinePointerInfo(Arg0),
8885 MachinePointerInfo(Arg1));
8886 if (Res.first.getNode()) {
8887 processIntegerCallValue(I, Res.first, true);
8888 PendingLoads.push_back(Res.second);
8889 return true;
8890 }
8891
8892 return false;
8893}
8894
8895/// See if we can lower a strlen call into an optimized form. If so, return
8896/// true and lower it, otherwise return false and it will be lowered like a
8897/// normal call.
8898/// The caller already checked that \p I calls the appropriate LibFunc with a
8899/// correct prototype.
8900bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
8901 const Value *Arg0 = I.getArgOperand(0);
8902
8904 std::pair<SDValue, SDValue> Res =
8906 getValue(Arg0), MachinePointerInfo(Arg0));
8907 if (Res.first.getNode()) {
8908 processIntegerCallValue(I, Res.first, false);
8909 PendingLoads.push_back(Res.second);
8910 return true;
8911 }
8912
8913 return false;
8914}
8915
8916/// See if we can lower a strnlen call into an optimized form. If so, return
8917/// true and lower it, otherwise return false and it will be lowered like a
8918/// normal call.
8919/// The caller already checked that \p I calls the appropriate LibFunc with a
8920/// correct prototype.
8921bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
8922 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
8923
8925 std::pair<SDValue, SDValue> Res =
8927 getValue(Arg0), getValue(Arg1),
8928 MachinePointerInfo(Arg0));
8929 if (Res.first.getNode()) {
8930 processIntegerCallValue(I, Res.first, false);
8931 PendingLoads.push_back(Res.second);
8932 return true;
8933 }
8934
8935 return false;
8936}
8937
8938/// See if we can lower a unary floating-point operation into an SDNode with
8939/// the specified Opcode. If so, return true and lower it, otherwise return
8940/// false and it will be lowered like a normal call.
8941/// The caller already checked that \p I calls the appropriate LibFunc with a
8942/// correct prototype.
8943bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
8944 unsigned Opcode) {
8945 // We already checked this call's prototype; verify it doesn't modify errno.
8946 if (!I.onlyReadsMemory())
8947 return false;
8948
8950 Flags.copyFMF(cast<FPMathOperator>(I));
8951
8952 SDValue Tmp = getValue(I.getArgOperand(0));
8953 setValue(&I,
8954 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
8955 return true;
8956}
8957
8958/// See if we can lower a binary floating-point operation into an SDNode with
8959/// the specified Opcode. If so, return true and lower it. Otherwise return
8960/// false, and it will be lowered like a normal call.
8961/// The caller already checked that \p I calls the appropriate LibFunc with a
8962/// correct prototype.
8963bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
8964 unsigned Opcode) {
8965 // We already checked this call's prototype; verify it doesn't modify errno.
8966 if (!I.onlyReadsMemory())
8967 return false;
8968
8970 Flags.copyFMF(cast<FPMathOperator>(I));
8971
8972 SDValue Tmp0 = getValue(I.getArgOperand(0));
8973 SDValue Tmp1 = getValue(I.getArgOperand(1));
8974 EVT VT = Tmp0.getValueType();
8975 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
8976 return true;
8977}
8978
8979void SelectionDAGBuilder::visitCall(const CallInst &I) {
8980 // Handle inline assembly differently.
8981 if (I.isInlineAsm()) {
8982 visitInlineAsm(I);
8983 return;
8984 }
8985
8987
8988 if (Function *F = I.getCalledFunction()) {
8989 if (F->isDeclaration()) {
8990 // Is this an LLVM intrinsic or a target-specific intrinsic?
8991 unsigned IID = F->getIntrinsicID();
8992 if (!IID)
8993 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
8994 IID = II->getIntrinsicID(F);
8995
8996 if (IID) {
8997 visitIntrinsicCall(I, IID);
8998 return;
8999 }
9000 }
9001
9002 // Check for well-known libc/libm calls. If the function is internal, it
9003 // can't be a library call. Don't do the check if marked as nobuiltin for
9004 // some reason or the call site requires strict floating point semantics.
9005 LibFunc Func;
9006 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
9007 F->hasName() && LibInfo->getLibFunc(*F, Func) &&
9009 switch (Func) {
9010 default: break;
9011 case LibFunc_bcmp:
9012 if (visitMemCmpBCmpCall(I))
9013 return;
9014 break;
9015 case LibFunc_copysign:
9016 case LibFunc_copysignf:
9017 case LibFunc_copysignl:
9018 // We already checked this call's prototype; verify it doesn't modify
9019 // errno.
9020 if (I.onlyReadsMemory()) {
9021 SDValue LHS = getValue(I.getArgOperand(0));
9022 SDValue RHS = getValue(I.getArgOperand(1));
9024 LHS.getValueType(), LHS, RHS));
9025 return;
9026 }
9027 break;
9028 case LibFunc_fabs:
9029 case LibFunc_fabsf:
9030 case LibFunc_fabsl:
9031 if (visitUnaryFloatCall(I, ISD::FABS))
9032 return;
9033 break;
9034 case LibFunc_fmin:
9035 case LibFunc_fminf:
9036 case LibFunc_fminl:
9037 if (visitBinaryFloatCall(I, ISD::FMINNUM))
9038 return;
9039 break;
9040 case LibFunc_fmax:
9041 case LibFunc_fmaxf:
9042 case LibFunc_fmaxl:
9043 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
9044 return;
9045 break;
9046 case LibFunc_sin:
9047 case LibFunc_sinf:
9048 case LibFunc_sinl:
9049 if (visitUnaryFloatCall(I, ISD::FSIN))
9050 return;
9051 break;
9052 case LibFunc_cos:
9053 case LibFunc_cosf:
9054 case LibFunc_cosl:
9055 if (visitUnaryFloatCall(I, ISD::FCOS))
9056 return;
9057 break;
9058 case LibFunc_sqrt:
9059 case LibFunc_sqrtf:
9060 case LibFunc_sqrtl:
9061 case LibFunc_sqrt_finite:
9062 case LibFunc_sqrtf_finite:
9063 case LibFunc_sqrtl_finite:
9064 if (visitUnaryFloatCall(I, ISD::FSQRT))
9065 return;
9066 break;
9067 case LibFunc_floor:
9068 case LibFunc_floorf:
9069 case LibFunc_floorl:
9070 if (visitUnaryFloatCall(I, ISD::FFLOOR))
9071 return;
9072 break;
9073 case LibFunc_nearbyint:
9074 case LibFunc_nearbyintf:
9075 case LibFunc_nearbyintl:
9076 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
9077 return;
9078 break;
9079 case LibFunc_ceil:
9080 case LibFunc_ceilf:
9081 case LibFunc_ceill:
9082 if (visitUnaryFloatCall(I, ISD::FCEIL))
9083 return;
9084 break;
9085 case LibFunc_rint:
9086 case LibFunc_rintf:
9087 case LibFunc_rintl:
9088 if (visitUnaryFloatCall(I, ISD::FRINT))
9089 return;
9090 break;
9091 case LibFunc_round:
9092 case LibFunc_roundf:
9093 case LibFunc_roundl:
9094 if (visitUnaryFloatCall(I, ISD::FROUND))
9095 return;
9096 break;
9097 case LibFunc_trunc:
9098 case LibFunc_truncf:
9099 case LibFunc_truncl:
9100 if (visitUnaryFloatCall(I, ISD::FTRUNC))
9101 return;
9102 break;
9103 case LibFunc_log2:
9104 case LibFunc_log2f:
9105 case LibFunc_log2l:
9106 if (visitUnaryFloatCall(I, ISD::FLOG2))
9107 return;
9108 break;
9109 case LibFunc_exp2:
9110 case LibFunc_exp2f:
9111 case LibFunc_exp2l:
9112 if (visitUnaryFloatCall(I, ISD::FEXP2))
9113 return;
9114 break;
9115 case LibFunc_exp10:
9116 case LibFunc_exp10f:
9117 case LibFunc_exp10l:
9118 if (visitUnaryFloatCall(I, ISD::FEXP10))
9119 return;
9120 break;
9121 case LibFunc_ldexp:
9122 case LibFunc_ldexpf:
9123 case LibFunc_ldexpl:
9124 if (visitBinaryFloatCall(I, ISD::FLDEXP))
9125 return;
9126 break;
9127 case LibFunc_memcmp:
9128 if (visitMemCmpBCmpCall(I))
9129 return;
9130 break;
9131 case LibFunc_mempcpy:
9132 if (visitMemPCpyCall(I))
9133 return;
9134 break;
9135 case LibFunc_memchr:
9136 if (visitMemChrCall(I))
9137 return;
9138 break;
9139 case LibFunc_strcpy:
9140 if (visitStrCpyCall(I, false))
9141 return;
9142 break;
9143 case LibFunc_stpcpy:
9144 if (visitStrCpyCall(I, true))
9145 return;
9146 break;
9147 case LibFunc_strcmp:
9148 if (visitStrCmpCall(I))
9149 return;
9150 break;
9151 case LibFunc_strlen:
9152 if (visitStrLenCall(I))
9153 return;
9154 break;
9155 case LibFunc_strnlen:
9156 if (visitStrNLenCall(I))
9157 return;
9158 break;
9159 }
9160 }
9161 }
9162
9163 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9164 // have to do anything here to lower funclet bundles.
9165 // CFGuardTarget bundles are lowered in LowerCallTo.
9166 assert(!I.hasOperandBundlesOtherThan(
9167 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9168 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9169 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9170 LLVMContext::OB_convergencectrl}) &&
9171 "Cannot lower calls with arbitrary operand bundles!");
9172
9173 SDValue Callee = getValue(I.getCalledOperand());
9174
9175 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
9176 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
9177 else
9178 // Check if we can potentially perform a tail call. More detailed checking
9179 // is be done within LowerCallTo, after more information about the call is
9180 // known.
9181 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
9182}
9183
9184namespace {
9185
9186/// AsmOperandInfo - This contains information for each constraint that we are
9187/// lowering.
9188class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9189public:
9190 /// CallOperand - If this is the result output operand or a clobber
9191 /// this is null, otherwise it is the incoming operand to the CallInst.
9192 /// This gets modified as the asm is processed.
9193 SDValue CallOperand;
9194
9195 /// AssignedRegs - If this is a register or register class operand, this
9196 /// contains the set of register corresponding to the operand.
9197 RegsForValue AssignedRegs;
9198
9199 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9200 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9201 }
9202
9203 /// Whether or not this operand accesses memory
9204 bool hasMemory(const TargetLowering &TLI) const {
9205 // Indirect operand accesses access memory.
9206 if (isIndirect)
9207 return true;
9208
9209 for (const auto &Code : Codes)
9211 return true;
9212
9213 return false;
9214 }
9215};
9216
9217
9218} // end anonymous namespace
9219
9220/// Make sure that the output operand \p OpInfo and its corresponding input
9221/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9222/// out).
9223static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9224 SDISelAsmOperandInfo &MatchingOpInfo,
9225 SelectionDAG &DAG) {
9226 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9227 return;
9228
9230 const auto &TLI = DAG.getTargetLoweringInfo();
9231
9232 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9233 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
9234 OpInfo.ConstraintVT);
9235 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9236 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
9237 MatchingOpInfo.ConstraintVT);
9238 if ((OpInfo.ConstraintVT.isInteger() !=
9239 MatchingOpInfo.ConstraintVT.isInteger()) ||
9240 (MatchRC.second != InputRC.second)) {
9241 // FIXME: error out in a more elegant fashion
9242 report_fatal_error("Unsupported asm: input constraint"
9243 " with a matching output constraint of"
9244 " incompatible type!");
9245 }
9246 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9247}
9248
9249/// Get a direct memory input to behave well as an indirect operand.
9250/// This may introduce stores, hence the need for a \p Chain.
9251/// \return The (possibly updated) chain.
9252static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9253 SDISelAsmOperandInfo &OpInfo,
9254 SelectionDAG &DAG) {
9255 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9256
9257 // If we don't have an indirect input, put it in the constpool if we can,
9258 // otherwise spill it to a stack slot.
9259 // TODO: This isn't quite right. We need to handle these according to
9260 // the addressing mode that the constraint wants. Also, this may take
9261 // an additional register for the computation and we don't want that
9262 // either.
9263
9264 // If the operand is a float, integer, or vector constant, spill to a
9265 // constant pool entry to get its address.
9266 const Value *OpVal = OpInfo.CallOperandVal;
9267 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
9268 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
9269 OpInfo.CallOperand = DAG.getConstantPool(
9270 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
9271 return Chain;
9272 }
9273
9274 // Otherwise, create a stack slot and emit a store to it before the asm.
9275 Type *Ty = OpVal->getType();
9276 auto &DL = DAG.getDataLayout();
9277 uint64_t TySize = DL.getTypeAllocSize(Ty);
9279 int SSFI = MF.getFrameInfo().CreateStackObject(
9280 TySize, DL.getPrefTypeAlign(Ty), false);
9281 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
9282 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
9284 TLI.getMemValueType(DL, Ty));
9285 OpInfo.CallOperand = StackSlot;
9286
9287 return Chain;
9288}
9289
9290/// GetRegistersForValue - Assign registers (virtual or physical) for the
9291/// specified operand. We prefer to assign virtual registers, to allow the
9292/// register allocator to handle the assignment process. However, if the asm
9293/// uses features that we can't model on machineinstrs, we have SDISel do the
9294/// allocation. This produces generally horrible, but correct, code.
9295///
9296/// OpInfo describes the operand
9297/// RefOpInfo describes the matching operand if any, the operand otherwise
9298static std::optional<unsigned>
9300 SDISelAsmOperandInfo &OpInfo,
9301 SDISelAsmOperandInfo &RefOpInfo) {
9302 LLVMContext &Context = *DAG.getContext();
9303 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9304
9308
9309 // No work to do for memory/address operands.
9310 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9311 OpInfo.ConstraintType == TargetLowering::C_Address)
9312 return std::nullopt;
9313
9314 // If this is a constraint for a single physreg, or a constraint for a
9315 // register class, find it.
9316 unsigned AssignedReg;
9317 const TargetRegisterClass *RC;
9318 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
9319 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
9320 // RC is unset only on failure. Return immediately.
9321 if (!RC)
9322 return std::nullopt;
9323
9324 // Get the actual register value type. This is important, because the user
9325 // may have asked for (e.g.) the AX register in i32 type. We need to
9326 // remember that AX is actually i16 to get the right extension.
9327 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
9328
9329 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9330 // If this is an FP operand in an integer register (or visa versa), or more
9331 // generally if the operand value disagrees with the register class we plan
9332 // to stick it in, fix the operand type.
9333 //
9334 // If this is an input value, the bitcast to the new type is done now.
9335 // Bitcast for output value is done at the end of visitInlineAsm().
9336 if ((OpInfo.Type == InlineAsm::isOutput ||
9337 OpInfo.Type == InlineAsm::isInput) &&
9338 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
9339 // Try to convert to the first EVT that the reg class contains. If the
9340 // types are identical size, use a bitcast to convert (e.g. two differing
9341 // vector types). Note: output bitcast is done at the end of
9342 // visitInlineAsm().
9343 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9344 // Exclude indirect inputs while they are unsupported because the code
9345 // to perform the load is missing and thus OpInfo.CallOperand still
9346 // refers to the input address rather than the pointed-to value.
9347 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
9348 OpInfo.CallOperand =
9349 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
9350 OpInfo.ConstraintVT = RegVT;
9351 // If the operand is an FP value and we want it in integer registers,
9352 // use the corresponding integer type. This turns an f64 value into
9353 // i64, which can be passed with two i32 values on a 32-bit machine.
9354 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9355 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
9356 if (OpInfo.Type == InlineAsm::isInput)
9357 OpInfo.CallOperand =
9358 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
9359 OpInfo.ConstraintVT = VT;
9360 }
9361 }
9362 }
9363
9364 // No need to allocate a matching input constraint since the constraint it's
9365 // matching to has already been allocated.
9366 if (OpInfo.isMatchingInputConstraint())
9367 return std::nullopt;
9368
9369 EVT ValueVT = OpInfo.ConstraintVT;
9370 if (OpInfo.ConstraintVT == MVT::Other)
9371 ValueVT = RegVT;
9372
9373 // Initialize NumRegs.
9374 unsigned NumRegs = 1;
9375 if (OpInfo.ConstraintVT != MVT::Other)
9376 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
9377
9378 // If this is a constraint for a specific physical register, like {r17},
9379 // assign it now.
9380
9381 // If this associated to a specific register, initialize iterator to correct
9382 // place. If virtual, make sure we have enough registers
9383
9384 // Initialize iterator if necessary
9387
9388 // Do not check for single registers.
9389 if (AssignedReg) {
9390 I = std::find(I, RC->end(), AssignedReg);
9391 if (I == RC->end()) {
9392 // RC does not contain the selected register, which indicates a
9393 // mismatch between the register and the required type/bitwidth.
9394 return {AssignedReg};
9395 }
9396 }
9397
9398 for (; NumRegs; --NumRegs, ++I) {
9399 assert(I != RC->end() && "Ran out of registers to allocate!");
9400 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
9401 Regs.push_back(R);
9402 }
9403
9404 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
9405 return std::nullopt;
9406}
9407
9408static unsigned
9410 const std::vector<SDValue> &AsmNodeOperands) {
9411 // Scan until we find the definition we already emitted of this operand.
9412 unsigned CurOp = InlineAsm::Op_FirstOperand;
9413 for (; OperandNo; --OperandNo) {
9414 // Advance to the next operand.
9415 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9416 const InlineAsm::Flag F(OpFlag);
9417 assert(
9418 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
9419 "Skipped past definitions?");
9420 CurOp += F.getNumOperandRegisters() + 1;
9421 }
9422 return CurOp;
9423}
9424
9425namespace {
9426
9427class ExtraFlags {
9428 unsigned Flags = 0;
9429
9430public:
9431 explicit ExtraFlags(const CallBase &Call) {
9432 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9433 if (IA->hasSideEffects())
9435 if (IA->isAlignStack())
9437 if (Call.isConvergent())
9439 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
9440 }
9441
9442 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
9443 // Ideally, we would only check against memory constraints. However, the
9444 // meaning of an Other constraint can be target-specific and we can't easily
9445 // reason about it. Therefore, be conservative and set MayLoad/MayStore
9446 // for Other constraints as well.
9449 if (OpInfo.Type == InlineAsm::isInput)
9451 else if (OpInfo.Type == InlineAsm::isOutput)
9453 else if (OpInfo.Type == InlineAsm::isClobber)
9455 }
9456 }
9457
9458 unsigned get() const { return Flags; }
9459};
9460
9461} // end anonymous namespace
9462
9463static bool isFunction(SDValue Op) {
9464 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
9465 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
9466 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
9467
9468 // In normal "call dllimport func" instruction (non-inlineasm) it force
9469 // indirect access by specifing call opcode. And usually specially print
9470 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
9471 // not do in this way now. (In fact, this is similar with "Data Access"
9472 // action). So here we ignore dllimport function.
9473 if (Fn && !Fn->hasDLLImportStorageClass())
9474 return true;
9475 }
9476 }
9477 return false;
9478}
9479
9480/// visitInlineAsm - Handle a call to an InlineAsm object.
9481void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
9482 const BasicBlock *EHPadBB) {
9483 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9484
9485 /// ConstraintOperands - Information about all of the constraints.
9486 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
9487
9491
9492 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
9493 // AsmDialect, MayLoad, MayStore).
9494 bool HasSideEffect = IA->hasSideEffects();
9495 ExtraFlags ExtraInfo(Call);
9496
9497 for (auto &T : TargetConstraints) {
9498 ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
9499 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
9500
9501 if (OpInfo.CallOperandVal)
9502 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
9503
9504 if (!HasSideEffect)
9505 HasSideEffect = OpInfo.hasMemory(TLI);
9506
9507 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
9508 // FIXME: Could we compute this on OpInfo rather than T?
9509
9510 // Compute the constraint code and ConstraintType to use.
9512
9513 if (T.ConstraintType == TargetLowering::C_Immediate &&
9514 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
9515 // We've delayed emitting a diagnostic like the "n" constraint because
9516 // inlining could cause an integer showing up.
9517 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
9518 "' expects an integer constant "
9519 "expression");
9520
9521 ExtraInfo.update(T);
9522 }
9523
9524 // We won't need to flush pending loads if this asm doesn't touch
9525 // memory and is nonvolatile.
9526 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
9527
9528 bool EmitEHLabels = isa<InvokeInst>(Call);
9529 if (EmitEHLabels) {
9530 assert(EHPadBB && "InvokeInst must have an EHPadBB");
9531 }
9532 bool IsCallBr = isa<CallBrInst>(Call);
9533
9534 if (IsCallBr || EmitEHLabels) {
9535 // If this is a callbr or invoke we need to flush pending exports since
9536 // inlineasm_br and invoke are terminators.
9537 // We need to do this before nodes are glued to the inlineasm_br node.
9538 Chain = getControlRoot();
9539 }
9540
9541 MCSymbol *BeginLabel = nullptr;
9542 if (EmitEHLabels) {
9543 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
9544 }
9545
9546 int OpNo = -1;
9547 SmallVector<StringRef> AsmStrs;
9548 IA->collectAsmStrs(AsmStrs);
9549
9550 // Second pass over the constraints: compute which constraint option to use.
9551 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9552 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
9553 OpNo++;
9554
9555 // If this is an output operand with a matching input operand, look up the
9556 // matching input. If their types mismatch, e.g. one is an integer, the
9557 // other is floating point, or their sizes are different, flag it as an
9558 // error.
9559 if (OpInfo.hasMatchingInput()) {
9560 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
9561 patchMatchingInput(OpInfo, Input, DAG);
9562 }
9563
9564 // Compute the constraint code and ConstraintType to use.
9565 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
9566
9567 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
9568 OpInfo.Type == InlineAsm::isClobber) ||
9569 OpInfo.ConstraintType == TargetLowering::C_Address)
9570 continue;
9571
9572 // In Linux PIC model, there are 4 cases about value/label addressing:
9573 //
9574 // 1: Function call or Label jmp inside the module.
9575 // 2: Data access (such as global variable, static variable) inside module.
9576 // 3: Function call or Label jmp outside the module.
9577 // 4: Data access (such as global variable) outside the module.
9578 //
9579 // Due to current llvm inline asm architecture designed to not "recognize"
9580 // the asm code, there are quite troubles for us to treat mem addressing
9581 // differently for same value/adress used in different instuctions.
9582 // For example, in pic model, call a func may in plt way or direclty
9583 // pc-related, but lea/mov a function adress may use got.
9584 //
9585 // Here we try to "recognize" function call for the case 1 and case 3 in
9586 // inline asm. And try to adjust the constraint for them.
9587 //
9588 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
9589 // label, so here we don't handle jmp function label now, but we need to
9590 // enhance it (especilly in PIC model) if we meet meaningful requirements.
9591 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
9592 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
9594 OpInfo.isIndirect = false;
9595 OpInfo.ConstraintType = TargetLowering::C_Address;
9596 }
9597
9598 // If this is a memory input, and if the operand is not indirect, do what we
9599 // need to provide an address for the memory input.
9600 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
9601 !OpInfo.isIndirect) {
9602 assert((OpInfo.isMultipleAlternative ||
9603 (OpInfo.Type == InlineAsm::isInput)) &&
9604 "Can only indirectify direct input operands!");
9605
9606 // Memory operands really want the address of the value.
9607 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
9608
9609 // There is no longer a Value* corresponding to this operand.
9610 OpInfo.CallOperandVal = nullptr;
9611
9612 // It is now an indirect operand.
9613 OpInfo.isIndirect = true;
9614 }
9615
9616 }
9617
9618 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
9619 std::vector<SDValue> AsmNodeOperands;
9620 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
9621 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
9622 IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
9623
9624 // If we have a !srcloc metadata node associated with it, we want to attach
9625 // this to the ultimately generated inline asm machineinstr. To do this, we
9626 // pass in the third operand as this (potentially null) inline asm MDNode.
9627 const MDNode *SrcLoc = Call.getMetadata("srcloc");
9628 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
9629
9630 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
9631 // bits as operand 3.
9632 AsmNodeOperands.push_back(DAG.getTargetConstant(
9633 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9634
9635 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
9636 // this, assign virtual and physical registers for inputs and otput.
9637 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9638 // Assign Registers.
9639 SDISelAsmOperandInfo &RefOpInfo =
9640 OpInfo.isMatchingInputConstraint()
9641 ? ConstraintOperands[OpInfo.getMatchedOperand()]
9642 : OpInfo;
9643 const auto RegError =
9644 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
9645 if (RegError) {
9648 const char *RegName = TRI.getName(*RegError);
9649 emitInlineAsmError(Call, "register '" + Twine(RegName) +
9650 "' allocated for constraint '" +
9651 Twine(OpInfo.ConstraintCode) +
9652 "' does not match required type");
9653 return;
9654 }
9655
9656 auto DetectWriteToReservedRegister = [&]() {
9659 for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
9661 TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
9662 const char *RegName = TRI.getName(Reg);
9663 emitInlineAsmError(Call, "write to reserved register '" +
9664 Twine(RegName) + "'");
9665 return true;
9666 }
9667 }
9668 return false;
9669 };
9670 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
9671 (OpInfo.Type == InlineAsm::isInput &&
9672 !OpInfo.isMatchingInputConstraint())) &&
9673 "Only address as input operand is allowed.");
9674
9675 switch (OpInfo.Type) {
9677 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9678 const InlineAsm::ConstraintCode ConstraintID =
9679 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9681 "Failed to convert memory constraint code to constraint id.");
9682
9683 // Add information to the INLINEASM node to know about this output.
9685 OpFlags.setMemConstraint(ConstraintID);
9686 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
9687 MVT::i32));
9688 AsmNodeOperands.push_back(OpInfo.CallOperand);
9689 } else {
9690 // Otherwise, this outputs to a register (directly for C_Register /
9691 // C_RegisterClass, and a target-defined fashion for
9692 // C_Immediate/C_Other). Find a register that we can use.
9693 if (OpInfo.AssignedRegs.Regs.empty()) {
9694 emitInlineAsmError(
9695 Call, "couldn't allocate output register for constraint '" +
9696 Twine(OpInfo.ConstraintCode) + "'");
9697 return;
9698 }
9699
9700 if (DetectWriteToReservedRegister())
9701 return;
9702
9703 // Add information to the INLINEASM node to know that this register is
9704 // set.
9705 OpInfo.AssignedRegs.AddInlineAsmOperands(
9706 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
9708 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
9709 }
9710 break;
9711
9712 case InlineAsm::isInput:
9713 case InlineAsm::isLabel: {
9714 SDValue InOperandVal = OpInfo.CallOperand;
9715
9716 if (OpInfo.isMatchingInputConstraint()) {
9717 // If this is required to match an output register we have already set,
9718 // just use its register.
9719 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
9720 AsmNodeOperands);
9721 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
9722 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
9723 if (OpInfo.isIndirect) {
9724 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
9725 emitInlineAsmError(Call, "inline asm not supported yet: "
9726 "don't know how to handle tied "
9727 "indirect register inputs");
9728 return;
9729 }
9730
9735 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
9736 Register TiedReg = R->getReg();
9737 MVT RegVT = R->getSimpleValueType(0);
9738 const TargetRegisterClass *RC =
9739 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)
9740 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
9741 : TRI.getMinimalPhysRegClass(TiedReg);
9742 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
9743 Regs.push_back(MRI.createVirtualRegister(RC));
9744
9745 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
9746
9747 SDLoc dl = getCurSDLoc();
9748 // Use the produced MatchedRegs object to
9749 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);
9750 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, true,
9751 OpInfo.getMatchedOperand(), dl, DAG,
9752 AsmNodeOperands);
9753 break;
9754 }
9755
9756 assert(Flag.isMemKind() && "Unknown matching constraint!");
9757 assert(Flag.getNumOperandRegisters() == 1 &&
9758 "Unexpected number of operands");
9759 // Add information to the INLINEASM node to know about this input.
9760 // See InlineAsm.h isUseOperandTiedToDef.
9761 Flag.clearMemConstraint();
9762 Flag.setMatchingOp(OpInfo.getMatchedOperand());
9763 AsmNodeOperands.push_back(DAG.getTargetConstant(
9764 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9765 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
9766 break;
9767 }
9768
9769 // Treat indirect 'X' constraint as memory.
9770 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
9771 OpInfo.isIndirect)
9772 OpInfo.ConstraintType = TargetLowering::C_Memory;
9773
9774 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
9775 OpInfo.ConstraintType == TargetLowering::C_Other) {
9776 std::vector<SDValue> Ops;
9777 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
9778 Ops, DAG);
9779 if (Ops.empty()) {
9780 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
9781 if (isa<ConstantSDNode>(InOperandVal)) {
9782 emitInlineAsmError(Call, "value out of range for constraint '" +
9783 Twine(OpInfo.ConstraintCode) + "'");
9784 return;
9785 }
9786
9787 emitInlineAsmError(Call,
9788 "invalid operand for inline asm constraint '" +
9789 Twine(OpInfo.ConstraintCode) + "'");
9790 return;
9791 }
9792
9793 // Add information to the INLINEASM node to know about this input.
9794 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
9795 AsmNodeOperands.push_back(DAG.getTargetConstant(
9796 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
9797 llvm::append_range(AsmNodeOperands, Ops);
9798 break;
9799 }
9800
9801 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
9802 assert((OpInfo.isIndirect ||
9803 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
9804 "Operand must be indirect to be a mem!");
9805 assert(InOperandVal.getValueType() ==
9807 "Memory operands expect pointer values");
9808
9809 const InlineAsm::ConstraintCode ConstraintID =
9810 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9812 "Failed to convert memory constraint code to constraint id.");
9813
9814 // Add information to the INLINEASM node to know about this input.
9816 ResOpType.setMemConstraint(ConstraintID);
9817 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
9818 getCurSDLoc(),
9819 MVT::i32));
9820 AsmNodeOperands.push_back(InOperandVal);
9821 break;
9822 }
9823
9824 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
9825 const InlineAsm::ConstraintCode ConstraintID =
9826 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
9828 "Failed to convert memory constraint code to constraint id.");
9829
9831
9832 SDValue AsmOp = InOperandVal;
9833 if (isFunction(InOperandVal)) {
9834 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
9835 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
9836 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
9837 InOperandVal.getValueType(),
9838 GA->getOffset());
9839 }
9840
9841 // Add information to the INLINEASM node to know about this input.
9842 ResOpType.setMemConstraint(ConstraintID);
9843
9844 AsmNodeOperands.push_back(
9845 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
9846
9847 AsmNodeOperands.push_back(AsmOp);
9848 break;
9849 }
9850
9851 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
9852 OpInfo.ConstraintType == TargetLowering::C_Register) &&
9853 "Unknown constraint type!");
9854
9855 // TODO: Support this.
9856 if (OpInfo.isIndirect) {
9857 emitInlineAsmError(
9858 Call, "Don't know how to handle indirect register inputs yet "
9859 "for constraint '" +
9860 Twine(OpInfo.ConstraintCode) + "'");
9861 return;
9862 }
9863
9864 // Copy the input into the appropriate registers.
9865 if (OpInfo.AssignedRegs.Regs.empty()) {
9866 emitInlineAsmError(Call,
9867 "couldn't allocate input reg for constraint '" +
9868 Twine(OpInfo.ConstraintCode) + "'");
9869 return;
9870 }
9871
9872 if (DetectWriteToReservedRegister())
9873 return;
9874
9875 SDLoc dl = getCurSDLoc();
9876
9877 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,
9878 &Call);
9879
9880 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, false,
9881 0, dl, DAG, AsmNodeOperands);
9882 break;
9883 }
9885 // Add the clobbered value to the operand list, so that the register
9886 // allocator is aware that the physreg got clobbered.
9887 if (!OpInfo.AssignedRegs.Regs.empty())
9888 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::Clobber,
9889 false, 0, getCurSDLoc(), DAG,
9890 AsmNodeOperands);
9891 break;
9892 }
9893 }
9894
9895 // Finish up input operands. Set the input chain and add the flag last.
9896 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
9897 if (Glue.getNode()) AsmNodeOperands.push_back(Glue);
9898
9899 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
9900 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
9901 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
9902 Glue = Chain.getValue(1);
9903
9904 // Do additional work to generate outputs.
9905
9906 SmallVector<EVT, 1> ResultVTs;
9907 SmallVector<SDValue, 1> ResultValues;
9908 SmallVector<SDValue, 8> OutChains;
9909
9910 llvm::Type *CallResultType = Call.getType();
9911 ArrayRef<Type *> ResultTypes;
9912 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
9913 ResultTypes = StructResult->elements();
9914 else if (!CallResultType->isVoidTy())
9915 ResultTypes = ArrayRef(CallResultType);
9916
9917 auto CurResultType = ResultTypes.begin();
9918 auto handleRegAssign = [&](SDValue V) {
9919 assert(CurResultType != ResultTypes.end() && "Unexpected value");
9920 assert((*CurResultType)->isSized() && "Unexpected unsized type");
9921 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
9922 ++CurResultType;
9923 // If the type of the inline asm call site return value is different but has
9924 // same size as the type of the asm output bitcast it. One example of this
9925 // is for vectors with different width / number of elements. This can
9926 // happen for register classes that can contain multiple different value
9927 // types. The preg or vreg allocated may not have the same VT as was
9928 // expected.
9929 //
9930 // This can also happen for a return value that disagrees with the register
9931 // class it is put in, eg. a double in a general-purpose register on a
9932 // 32-bit machine.
9933 if (ResultVT != V.getValueType() &&
9934 ResultVT.getSizeInBits() == V.getValueSizeInBits())
9935 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
9936 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
9937 V.getValueType().isInteger()) {
9938 // If a result value was tied to an input value, the computed result
9939 // may have a wider width than the expected result. Extract the
9940 // relevant portion.
9941 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
9942 }
9943 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
9944 ResultVTs.push_back(ResultVT);
9945 ResultValues.push_back(V);
9946 };
9947
9948 // Deal with output operands.
9949 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9950 if (OpInfo.Type == InlineAsm::isOutput) {
9951 SDValue Val;
9952 // Skip trivial output operands.
9953 if (OpInfo.AssignedRegs.Regs.empty())
9954 continue;
9955
9956 switch (OpInfo.ConstraintType) {
9959 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
9960 Chain, &Glue, &Call);
9961 break;
9964 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
9965 OpInfo, DAG);
9966 break;
9968 break; // Already handled.
9970 break; // Silence warning.
9972 assert(false && "Unexpected unknown constraint");
9973 }
9974
9975 // Indirect output manifest as stores. Record output chains.
9976 if (OpInfo.isIndirect) {
9977 const Value *Ptr = OpInfo.CallOperandVal;
9978 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
9979 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
9981 OutChains.push_back(Store);
9982 } else {
9983 // generate CopyFromRegs to associated registers.
9984 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
9985 if (Val.getOpcode() == ISD::MERGE_VALUES) {
9986 for (const SDValue &V : Val->op_values())
9987 handleRegAssign(V);
9988 } else
9989 handleRegAssign(Val);
9990 }
9991 }
9992 }
9993
9994 // Set results.
9995 if (!ResultValues.empty()) {
9996 assert(CurResultType == ResultTypes.end() &&
9997 "Mismatch in number of ResultTypes");
9998 assert(ResultValues.size() == ResultTypes.size() &&
9999 "Mismatch in number of output operands in asm result");
10000
10002 DAG.getVTList(ResultVTs), ResultValues);
10003 setValue(&Call, V);
10004 }
10005
10006 // Collect store chains.
10007 if (!OutChains.empty())
10008 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
10009
10010 if (EmitEHLabels) {
10011 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
10012 }
10013
10014 // Only Update Root if inline assembly has a memory effect.
10015 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10016 EmitEHLabels)
10017 DAG.setRoot(Chain);
10018}
10019
10020void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10021 const Twine &Message) {
10022 LLVMContext &Ctx = *DAG.getContext();
10023 Ctx.emitError(&Call, Message);
10024
10025 // Make sure we leave the DAG in a valid state
10027 SmallVector<EVT, 1> ValueVTs;
10028 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
10029
10030 if (ValueVTs.empty())
10031 return;
10032
10034 for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
10035 Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
10036
10037 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
10038}
10039
10040void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10042 MVT::Other, getRoot(),
10043 getValue(I.getArgOperand(0)),
10044 DAG.getSrcValue(I.getArgOperand(0))));
10045}
10046
10047void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10049 const DataLayout &DL = DAG.getDataLayout();
10051 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
10052 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
10053 DL.getABITypeAlign(I.getType()).value());
10054 DAG.setRoot(V.getValue(1));
10055
10056 if (I.getType()->isPointerTy())
10058 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
10059 setValue(&I, V);
10060}
10061
10062void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10064 MVT::Other, getRoot(),
10065 getValue(I.getArgOperand(0)),
10066 DAG.getSrcValue(I.getArgOperand(0))));
10067}
10068
10069void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10071 MVT::Other, getRoot(),
10072 getValue(I.getArgOperand(0)),
10073 getValue(I.getArgOperand(1)),
10074 DAG.getSrcValue(I.getArgOperand(0)),
10075 DAG.getSrcValue(I.getArgOperand(1))));
10076}
10077
10079 const Instruction &I,
10080 SDValue Op) {
10081 const MDNode *Range = getRangeMetadata(I);
10082 if (!Range)
10083 return Op;
10084
10086 if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
10087 return Op;
10088
10089 APInt Lo = CR.getUnsignedMin();
10090 if (!Lo.isMinValue())
10091 return Op;
10092
10093 APInt Hi = CR.getUnsignedMax();
10094 unsigned Bits = std::max(Hi.getActiveBits(),
10095 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10096
10097 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
10098
10099 SDLoc SL = getCurSDLoc();
10100
10101 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
10102 DAG.getValueType(SmallVT));
10103 unsigned NumVals = Op.getNode()->getNumValues();
10104 if (NumVals == 1)
10105 return ZExt;
10106
10108
10109 Ops.push_back(ZExt);
10110 for (unsigned I = 1; I != NumVals; ++I)
10111 Ops.push_back(Op.getValue(I));
10112
10113 return DAG.getMergeValues(Ops, SL);
10114}
10115
10116/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10117/// the call being lowered.
10118///
10119/// This is a helper for lowering intrinsics that follow a target calling
10120/// convention or require stack pointer adjustment. Only a subset of the
10121/// intrinsic's operands need to participate in the calling convention.
10124 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10125 AttributeSet RetAttrs, bool IsPatchPoint) {
10127 Args.reserve(NumArgs);
10128
10129 // Populate the argument list.
10130 // Attributes for args start at offset 1, after the return attribute.
10131 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10132 ArgI != ArgE; ++ArgI) {
10133 const Value *V = Call->getOperand(ArgI);
10134
10135 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10136
10138 Entry.Node = getValue(V);
10139 Entry.Ty = V->getType();
10140 Entry.setAttributes(Call, ArgI);
10141 Args.push_back(Entry);
10142 }
10143
10145 .setChain(getRoot())
10146 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
10147 RetAttrs)
10148 .setDiscardResult(Call->use_empty())
10149 .setIsPatchPoint(IsPatchPoint)
10151 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
10152}
10153
10154/// Add a stack map intrinsic call's live variable operands to a stackmap
10155/// or patchpoint target node's operand list.
10156///
10157/// Constants are converted to TargetConstants purely as an optimization to
10158/// avoid constant materialization and register allocation.
10159///
10160/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10161/// generate addess computation nodes, and so FinalizeISel can convert the
10162/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10163/// address materialization and register allocation, but may also be required
10164/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10165/// alloca in the entry block, then the runtime may assume that the alloca's
10166/// StackMap location can be read immediately after compilation and that the
10167/// location is valid at any point during execution (this is similar to the
10168/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10169/// only available in a register, then the runtime would need to trap when
10170/// execution reaches the StackMap in order to read the alloca's location.
10171static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10172 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
10173 SelectionDAGBuilder &Builder) {
10174 SelectionDAG &DAG = Builder.DAG;
10175 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10176 SDValue Op = Builder.getValue(Call.getArgOperand(I));
10177
10178 // Things on the stack are pointer-typed, meaning that they are already
10179 // legal and can be emitted directly to target nodes.
10180 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
10181 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
10182 } else {
10183 // Otherwise emit a target independent node to be legalised.
10184 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
10185 }
10186 }
10187}
10188
10189/// Lower llvm.experimental.stackmap.
10190void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10191 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10192 // [live variables...])
10193
10194 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10195
10196 SDValue Chain, InGlue, Callee;
10198
10199 SDLoc DL = getCurSDLoc();
10201
10202 // The stackmap intrinsic only records the live variables (the arguments
10203 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10204 // intrinsic, this won't be lowered to a function call. This means we don't
10205 // have to worry about calling conventions and target specific lowering code.
10206 // Instead we perform the call lowering right here.
10207 //
10208 // chain, flag = CALLSEQ_START(chain, 0, 0)
10209 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10210 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10211 //
10212 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
10213 InGlue = Chain.getValue(1);
10214
10215 // Add the STACKMAP operands, starting with DAG house-keeping.
10216 Ops.push_back(Chain);
10217 Ops.push_back(InGlue);
10218
10219 // Add the <id>, <numShadowBytes> operands.
10220 //
10221 // These do not require legalisation, and can be emitted directly to target
10222 // constant nodes.
10224 assert(ID.getValueType() == MVT::i64);
10225 SDValue IDConst =
10226 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
10227 Ops.push_back(IDConst);
10228
10229 SDValue Shad = getValue(CI.getArgOperand(1));
10230 assert(Shad.getValueType() == MVT::i32);
10231 SDValue ShadConst =
10233 Ops.push_back(ShadConst);
10234
10235 // Add the live variables.
10236 addStackMapLiveVars(CI, 2, DL, Ops, *this);
10237
10238 // Create the STACKMAP node.
10239 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10240 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
10241 InGlue = Chain.getValue(1);
10242
10243 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
10244
10245 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10246
10247 // Set the root to the target-lowered call chain.
10248 DAG.setRoot(Chain);
10249
10250 // Inform the Frame Information that we have a stackmap in this function.
10252}
10253
10254/// Lower llvm.experimental.patchpoint directly to its target opcode.
10255void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10256 const BasicBlock *EHPadBB) {
10257 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
10258 // i32 <numBytes>,
10259 // i8* <target>,
10260 // i32 <numArgs>,
10261 // [Args...],
10262 // [live variables...])
10263
10265 bool IsAnyRegCC = CC == CallingConv::AnyReg;
10266 bool HasDef = !CB.getType()->isVoidTy();
10267 SDLoc dl = getCurSDLoc();
10269
10270 // Handle immediate and symbolic callees.
10271 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
10272 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
10273 /*isTarget=*/true);
10274 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
10275 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
10276 SDLoc(SymbolicCallee),
10277 SymbolicCallee->getValueType(0));
10278
10279 // Get the real number of arguments participating in the call <numArgs>
10281 unsigned NumArgs = NArgVal->getAsZExtVal();
10282
10283 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
10284 // Intrinsics include all meta-operands up to but not including CC.
10285 unsigned NumMetaOpers = PatchPointOpers::CCPos;
10286 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
10287 "Not enough arguments provided to the patchpoint intrinsic");
10288
10289 // For AnyRegCC the arguments are lowered later on manually.
10290 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10291 Type *ReturnTy =
10292 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
10293
10295 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
10296 ReturnTy, CB.getAttributes().getRetAttrs(), true);
10297 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
10298
10299 SDNode *CallEnd = Result.second.getNode();
10300 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
10301 CallEnd = CallEnd->getOperand(0).getNode();
10302
10303 /// Get a call instruction from the call sequence chain.
10304 /// Tail calls are not allowed.
10305 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
10306 "Expected a callseq node.");
10307 SDNode *Call = CallEnd->getOperand(0).getNode();
10308 bool HasGlue = Call->getGluedNode();
10309
10310 // Replace the target specific call node with the patchable intrinsic.
10312
10313 // Push the chain.
10314 Ops.push_back(*(Call->op_begin()));
10315
10316 // Optionally, push the glue (if any).
10317 if (HasGlue)
10318 Ops.push_back(*(Call->op_end() - 1));
10319
10320 // Push the register mask info.
10321 if (HasGlue)
10322 Ops.push_back(*(Call->op_end() - 2));
10323 else
10324 Ops.push_back(*(Call->op_end() - 1));
10325
10326 // Add the <id> and <numBytes> constants.
10328 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
10330 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
10331
10332 // Add the callee.
10333 Ops.push_back(Callee);
10334
10335 // Adjust <numArgs> to account for any arguments that have been passed on the
10336 // stack instead.
10337 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
10338 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
10339 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10340 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
10341
10342 // Add the calling convention
10343 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
10344
10345 // Add the arguments we omitted previously. The register allocator should
10346 // place these in any free register.
10347 if (IsAnyRegCC)
10348 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
10349 Ops.push_back(getValue(CB.getArgOperand(i)));
10350
10351 // Push the arguments from the call instruction.
10352 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
10353 Ops.append(Call->op_begin() + 2, e);
10354
10355 // Push live variables for the stack map.
10356 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
10357
10358 SDVTList NodeTys;
10359 if (IsAnyRegCC && HasDef) {
10360 // Create the return types based on the intrinsic definition
10362 SmallVector<EVT, 3> ValueVTs;
10363 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
10364 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
10365
10366 // There is always a chain and a glue type at the end
10367 ValueVTs.push_back(MVT::Other);
10368 ValueVTs.push_back(MVT::Glue);
10369 NodeTys = DAG.getVTList(ValueVTs);
10370 } else
10371 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10372
10373 // Replace the target specific call node with a PATCHPOINT node.
10374 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
10375
10376 // Update the NodeMap.
10377 if (HasDef) {
10378 if (IsAnyRegCC)
10379 setValue(&CB, SDValue(PPV.getNode(), 0));
10380 else
10381 setValue(&CB, Result.first);
10382 }
10383
10384 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
10385 // call sequence. Furthermore the location of the chain and glue can change
10386 // when the AnyReg calling convention is used and the intrinsic returns a
10387 // value.
10388 if (IsAnyRegCC && HasDef) {
10389 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
10390 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
10392 } else
10393 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
10394 DAG.DeleteNode(Call);
10395
10396 // Inform the Frame Information that we have a patchpoint in this function.
10398}
10399
10400void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
10401 unsigned Intrinsic) {
10403 SDValue Op1 = getValue(I.getArgOperand(0));
10404 SDValue Op2;
10405 if (I.arg_size() > 1)
10406 Op2 = getValue(I.getArgOperand(1));
10407 SDLoc dl = getCurSDLoc();
10408 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
10409 SDValue Res;
10410 SDNodeFlags SDFlags;
10411 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
10412 SDFlags.copyFMF(*FPMO);
10413
10414 switch (Intrinsic) {
10415 case Intrinsic::vector_reduce_fadd:
10416 if (SDFlags.hasAllowReassociation())
10417 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
10418 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
10419 SDFlags);
10420 else
10421 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
10422 break;
10423 case Intrinsic::vector_reduce_fmul:
10424 if (SDFlags.hasAllowReassociation())
10425 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
10426 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
10427 SDFlags);
10428 else
10429 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
10430 break;
10431 case Intrinsic::vector_reduce_add:
10432 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
10433 break;
10434 case Intrinsic::vector_reduce_mul:
10435 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
10436 break;
10437 case Intrinsic::vector_reduce_and:
10438 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
10439 break;
10440 case Intrinsic::vector_reduce_or:
10441 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
10442 break;
10443 case Intrinsic::vector_reduce_xor:
10444 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
10445 break;
10446 case Intrinsic::vector_reduce_smax:
10447 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
10448 break;
10449 case Intrinsic::vector_reduce_smin:
10450 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
10451 break;
10452 case Intrinsic::vector_reduce_umax:
10453 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
10454 break;
10455 case Intrinsic::vector_reduce_umin:
10456 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
10457 break;
10458 case Intrinsic::vector_reduce_fmax:
10459 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
10460 break;
10461 case Intrinsic::vector_reduce_fmin:
10462 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
10463 break;
10464 case Intrinsic::vector_reduce_fmaximum:
10465 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
10466 break;
10467 case Intrinsic::vector_reduce_fminimum:
10468 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
10469 break;
10470 default:
10471 llvm_unreachable("Unhandled vector reduce intrinsic");
10472 }
10473 setValue(&I, Res);
10474}
10475
10476/// Returns an AttributeList representing the attributes applied to the return
10477/// value of the given call.
10480 if (CLI.RetSExt)
10481 Attrs.push_back(Attribute::SExt);
10482 if (CLI.RetZExt)
10483 Attrs.push_back(Attribute::ZExt);
10484 if (CLI.IsInReg)
10485 Attrs.push_back(Attribute::InReg);
10486
10488 Attrs);
10489}
10490
10491/// TargetLowering::LowerCallTo - This is the default LowerCallTo
10492/// implementation, which just calls LowerCall.
10493/// FIXME: When all targets are
10494/// migrated to using LowerCall, this hook should be integrated into SDISel.
10495std::pair<SDValue, SDValue>
10497 // Handle the incoming return values from the call.
10498 CLI.Ins.clear();
10499 Type *OrigRetTy = CLI.RetTy;
10500 SmallVector<EVT, 4> RetTys;
10502 auto &DL = CLI.DAG.getDataLayout();
10503 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets, 0);
10504
10505 if (CLI.IsPostTypeLegalization) {
10506 // If we are lowering a libcall after legalization, split the return type.
10507 SmallVector<EVT, 4> OldRetTys;
10508 SmallVector<uint64_t, 4> OldOffsets;
10509 RetTys.swap(OldRetTys);
10510 Offsets.swap(OldOffsets);
10511
10512 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
10513 EVT RetVT = OldRetTys[i];
10514 uint64_t Offset = OldOffsets[i];
10515 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
10516 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
10517 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
10518 RetTys.append(NumRegs, RegisterVT);
10519 for (unsigned j = 0; j != NumRegs; ++j)
10520 Offsets.push_back(Offset + j * RegisterVTByteSZ);
10521 }
10522 }
10523
10525 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
10526
10527 bool CanLowerReturn =
10529 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
10530
10531 SDValue DemoteStackSlot;
10532 int DemoteStackIdx = -100;
10533 if (!CanLowerReturn) {
10534 // FIXME: equivalent assert?
10535 // assert(!CS.hasInAllocaArgument() &&
10536 // "sret demotion is incompatible with inalloca");
10537 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
10538 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
10540 DemoteStackIdx =
10541 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
10542 Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
10543 DL.getAllocaAddrSpace());
10544
10545 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
10546 ArgListEntry Entry;
10547 Entry.Node = DemoteStackSlot;
10548 Entry.Ty = StackSlotPtrType;
10549 Entry.IsSExt = false;
10550 Entry.IsZExt = false;
10551 Entry.IsInReg = false;
10552 Entry.IsSRet = true;
10553 Entry.IsNest = false;
10554 Entry.IsByVal = false;
10555 Entry.IsByRef = false;
10556 Entry.IsReturned = false;
10557 Entry.IsSwiftSelf = false;
10558 Entry.IsSwiftAsync = false;
10559 Entry.IsSwiftError = false;
10560 Entry.IsCFGuardTarget = false;
10561 Entry.Alignment = Alignment;
10562 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
10563 CLI.NumFixedArgs += 1;
10564 CLI.getArgs()[0].IndirectType = CLI.RetTy;
10565 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
10566
10567 // sret demotion isn't compatible with tail-calls, since the sret argument
10568 // points into the callers stack frame.
10569 CLI.IsTailCall = false;
10570 } else {
10571 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10572 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
10573 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
10574 ISD::ArgFlagsTy Flags;
10575 if (NeedsRegBlock) {
10576 Flags.setInConsecutiveRegs();
10577 if (I == RetTys.size() - 1)
10578 Flags.setInConsecutiveRegsLast();
10579 }
10580 EVT VT = RetTys[I];
10582 CLI.CallConv, VT);
10583 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10584 CLI.CallConv, VT);
10585 for (unsigned i = 0; i != NumRegs; ++i) {
10586 ISD::InputArg MyFlags;
10587 MyFlags.Flags = Flags;
10588 MyFlags.VT = RegisterVT;
10589 MyFlags.ArgVT = VT;
10590 MyFlags.Used = CLI.IsReturnValueUsed;
10591 if (CLI.RetTy->isPointerTy()) {
10592 MyFlags.Flags.setPointer();
10593 MyFlags.Flags.setPointerAddrSpace(
10594 cast<PointerType>(CLI.RetTy)->getAddressSpace());
10595 }
10596 if (CLI.RetSExt)
10597 MyFlags.Flags.setSExt();
10598 if (CLI.RetZExt)
10599 MyFlags.Flags.setZExt();
10600 if (CLI.IsInReg)
10601 MyFlags.Flags.setInReg();
10602 CLI.Ins.push_back(MyFlags);
10603 }
10604 }
10605 }
10606
10607 // We push in swifterror return as the last element of CLI.Ins.
10608 ArgListTy &Args = CLI.getArgs();
10609 if (supportSwiftError()) {
10610 for (const ArgListEntry &Arg : Args) {
10611 if (Arg.IsSwiftError) {
10612 ISD::InputArg MyFlags;
10613 MyFlags.VT = getPointerTy(DL);
10614 MyFlags.ArgVT = EVT(getPointerTy(DL));
10615 MyFlags.Flags.setSwiftError();
10616 CLI.Ins.push_back(MyFlags);
10617 }
10618 }
10619 }
10620
10621 // Handle all of the outgoing arguments.
10622 CLI.Outs.clear();
10623 CLI.OutVals.clear();
10624 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
10625 SmallVector<EVT, 4> ValueVTs;
10626 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
10627 // FIXME: Split arguments if CLI.IsPostTypeLegalization
10628 Type *FinalType = Args[i].Ty;
10629 if (Args[i].IsByVal)
10630 FinalType = Args[i].IndirectType;
10631 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
10632 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
10633 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
10634 ++Value) {
10635 EVT VT = ValueVTs[Value];
10636 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
10637 SDValue Op = SDValue(Args[i].Node.getNode(),
10638 Args[i].Node.getResNo() + Value);
10639 ISD::ArgFlagsTy Flags;
10640
10641 // Certain targets (such as MIPS), may have a different ABI alignment
10642 // for a type depending on the context. Give the target a chance to
10643 // specify the alignment it wants.
10644 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
10645 Flags.setOrigAlign(OriginalAlignment);
10646
10647 if (Args[i].Ty->isPointerTy()) {
10648 Flags.setPointer();
10649 Flags.setPointerAddrSpace(
10650 cast<PointerType>(Args[i].Ty)->getAddressSpace());
10651 }
10652 if (Args[i].IsZExt)
10653 Flags.setZExt();
10654 if (Args[i].IsSExt)
10655 Flags.setSExt();
10656 if (Args[i].IsInReg) {
10657 // If we are using vectorcall calling convention, a structure that is
10658 // passed InReg - is surely an HVA
10660 isa<StructType>(FinalType)) {
10661 // The first value of a structure is marked
10662 if (0 == Value)
10663 Flags.setHvaStart();
10664 Flags.setHva();
10665 }
10666 // Set InReg Flag
10667 Flags.setInReg();
10668 }
10669 if (Args[i].IsSRet)
10670 Flags.setSRet();
10671 if (Args[i].IsSwiftSelf)
10672 Flags.setSwiftSelf();
10673 if (Args[i].IsSwiftAsync)
10674 Flags.setSwiftAsync();
10675 if (Args[i].IsSwiftError)
10676 Flags.setSwiftError();
10677 if (Args[i].IsCFGuardTarget)
10678 Flags.setCFGuardTarget();
10679 if (Args[i].IsByVal)
10680 Flags.setByVal();
10681 if (Args[i].IsByRef)
10682 Flags.setByRef();
10683 if (Args[i].IsPreallocated) {
10684 Flags.setPreallocated();
10685 // Set the byval flag for CCAssignFn callbacks that don't know about
10686 // preallocated. This way we can know how many bytes we should've
10687 // allocated and how many bytes a callee cleanup function will pop. If
10688 // we port preallocated to more targets, we'll have to add custom
10689 // preallocated handling in the various CC lowering callbacks.
10690 Flags.setByVal();
10691 }
10692 if (Args[i].IsInAlloca) {
10693 Flags.setInAlloca();
10694 // Set the byval flag for CCAssignFn callbacks that don't know about
10695 // inalloca. This way we can know how many bytes we should've allocated
10696 // and how many bytes a callee cleanup function will pop. If we port
10697 // inalloca to more targets, we'll have to add custom inalloca handling
10698 // in the various CC lowering callbacks.
10699 Flags.setByVal();
10700 }
10701 Align MemAlign;
10702 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
10703 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
10704 Flags.setByValSize(FrameSize);
10705
10706 // info is not there but there are cases it cannot get right.
10707 if (auto MA = Args[i].Alignment)
10708 MemAlign = *MA;
10709 else
10710 MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
10711 } else if (auto MA = Args[i].Alignment) {
10712 MemAlign = *MA;
10713 } else {
10714 MemAlign = OriginalAlignment;
10715 }
10716 Flags.setMemAlign(MemAlign);
10717 if (Args[i].IsNest)
10718 Flags.setNest();
10719 if (NeedsRegBlock)
10720 Flags.setInConsecutiveRegs();
10721
10723 CLI.CallConv, VT);
10724 unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10725 CLI.CallConv, VT);
10726 SmallVector<SDValue, 4> Parts(NumParts);
10727 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
10728
10729 if (Args[i].IsSExt)
10730 ExtendKind = ISD::SIGN_EXTEND;
10731 else if (Args[i].IsZExt)
10732 ExtendKind = ISD::ZERO_EXTEND;
10733
10734 // Conservatively only handle 'returned' on non-vectors that can be lowered,
10735 // for now.
10736 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
10738 assert((CLI.RetTy == Args[i].Ty ||
10739 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
10741 Args[i].Ty->getPointerAddressSpace())) &&
10742 RetTys.size() == NumValues && "unexpected use of 'returned'");
10743 // Before passing 'returned' to the target lowering code, ensure that
10744 // either the register MVT and the actual EVT are the same size or that
10745 // the return value and argument are extended in the same way; in these
10746 // cases it's safe to pass the argument register value unchanged as the
10747 // return register value (although it's at the target's option whether
10748 // to do so)
10749 // TODO: allow code generation to take advantage of partially preserved
10750 // registers rather than clobbering the entire register when the
10751 // parameter extension method is not compatible with the return
10752 // extension method
10753 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
10754 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
10755 CLI.RetZExt == Args[i].IsZExt))
10756 Flags.setReturned();
10757 }
10758
10759 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
10760 CLI.CallConv, ExtendKind);
10761
10762 for (unsigned j = 0; j != NumParts; ++j) {
10763 // if it isn't first piece, alignment must be 1
10764 // For scalable vectors the scalable part is currently handled
10765 // by individual targets, so we just use the known minimum size here.
10766 ISD::OutputArg MyFlags(
10767 Flags, Parts[j].getValueType().getSimpleVT(), VT,
10768 i < CLI.NumFixedArgs, i,
10769 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
10770 if (NumParts > 1 && j == 0)
10771 MyFlags.Flags.setSplit();
10772 else if (j != 0) {
10773 MyFlags.Flags.setOrigAlign(Align(1));
10774 if (j == NumParts - 1)
10775 MyFlags.Flags.setSplitEnd();
10776 }
10777
10778 CLI.Outs.push_back(MyFlags);
10779 CLI.OutVals.push_back(Parts[j]);
10780 }
10781
10782 if (NeedsRegBlock && Value == NumValues - 1)
10783 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
10784 }
10785 }
10786
10788 CLI.Chain = LowerCall(CLI, InVals);
10789
10790 // Update CLI.InVals to use outside of this function.
10791 CLI.InVals = InVals;
10792
10793 // Verify that the target's LowerCall behaved as expected.
10794 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
10795 "LowerCall didn't return a valid chain!");
10796 assert((!CLI.IsTailCall || InVals.empty()) &&
10797 "LowerCall emitted a return value for a tail call!");
10798 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
10799 "LowerCall didn't emit the correct number of values!");
10800
10801 // For a tail call, the return value is merely live-out and there aren't
10802 // any nodes in the DAG representing it. Return a special value to
10803 // indicate that a tail call has been emitted and no more Instructions
10804 // should be processed in the current block.
10805 if (CLI.IsTailCall) {
10806 CLI.DAG.setRoot(CLI.Chain);
10807 return std::make_pair(SDValue(), SDValue());
10808 }
10809
10810#ifndef NDEBUG
10811 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
10812 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
10813 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
10814 "LowerCall emitted a value with the wrong type!");
10815 }
10816#endif
10817
10818 SmallVector<SDValue, 4> ReturnValues;
10819 if (!CanLowerReturn) {
10820 // The instruction result is the result of loading from the
10821 // hidden sret parameter.
10823 Type *PtrRetTy =
10824 PointerType::get(OrigRetTy->getContext(), DL.getAllocaAddrSpace());
10825
10826 ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
10827 assert(PVTs.size() == 1 && "Pointers should fit in one register");
10828 EVT PtrVT = PVTs[0];
10829
10830 unsigned NumValues = RetTys.size();
10831 ReturnValues.resize(NumValues);
10832 SmallVector<SDValue, 4> Chains(NumValues);
10833
10834 // An aggregate return value cannot wrap around the address space, so
10835 // offsets to its parts don't wrap either.
10836 SDNodeFlags Flags;
10837 Flags.setNoUnsignedWrap(true);
10838
10840 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
10841 for (unsigned i = 0; i < NumValues; ++i) {
10842 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
10843 CLI.DAG.getConstant(Offsets[i], CLI.DL,
10844 PtrVT), Flags);
10845 SDValue L = CLI.DAG.getLoad(
10846 RetTys[i], CLI.DL, CLI.Chain, Add,
10848 DemoteStackIdx, Offsets[i]),
10849 HiddenSRetAlign);
10850 ReturnValues[i] = L;
10851 Chains[i] = L.getValue(1);
10852 }
10853
10854 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
10855 } else {
10856 // Collect the legal value parts into potentially illegal values
10857 // that correspond to the original function's return values.
10858 std::optional<ISD::NodeType> AssertOp;
10859 if (CLI.RetSExt)
10860 AssertOp = ISD::AssertSext;
10861 else if (CLI.RetZExt)
10862 AssertOp = ISD::AssertZext;
10863 unsigned CurReg = 0;
10864 for (EVT VT : RetTys) {
10866 CLI.CallConv, VT);
10867 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
10868 CLI.CallConv, VT);
10869
10870 ReturnValues.push_back(getCopyFromParts(
10871 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
10872 CLI.Chain, CLI.CallConv, AssertOp));
10873 CurReg += NumRegs;
10874 }
10875
10876 // For a function returning void, there is no return value. We can't create
10877 // such a node, so we just return a null return value in that case. In
10878 // that case, nothing will actually look at the value.
10879 if (ReturnValues.empty())
10880 return std::make_pair(SDValue(), CLI.Chain);
10881 }
10882
10883 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
10884 CLI.DAG.getVTList(RetTys), ReturnValues);
10885 return std::make_pair(Res, CLI.Chain);
10886}
10887
10888/// Places new result values for the node in Results (their number
10889/// and types must exactly match those of the original return values of
10890/// the node), or leaves Results empty, which indicates that the node is not
10891/// to be custom lowered after all.
10894 SelectionDAG &DAG) const {
10895 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
10896
10897 if (!Res.getNode())
10898 return;
10899
10900 // If the original node has one result, take the return value from
10901 // LowerOperation as is. It might not be result number 0.
10902 if (N->getNumValues() == 1) {
10903 Results.push_back(Res);
10904 return;
10905 }
10906
10907 // If the original node has multiple results, then the return node should
10908 // have the same number of results.
10909 assert((N->getNumValues() == Res->getNumValues()) &&
10910 "Lowering returned the wrong number of results!");
10911
10912 // Places new result values base on N result number.
10913 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
10914 Results.push_back(Res.getValue(I));
10915}
10916
10918 llvm_unreachable("LowerOperation not implemented for this target!");
10919}
10920
10922 unsigned Reg,
10923 ISD::NodeType ExtendType) {
10925 assert((Op.getOpcode() != ISD::CopyFromReg ||
10926 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
10927 "Copy from a reg to the same reg!");
10928 assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
10929
10931 // If this is an InlineAsm we have to match the registers required, not the
10932 // notional registers required by the type.
10933
10934 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
10935 std::nullopt); // This is not an ABI copy.
10936 SDValue Chain = DAG.getEntryNode();
10937
10938 if (ExtendType == ISD::ANY_EXTEND) {
10939 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
10940 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
10941 ExtendType = PreferredExtendIt->second;
10942 }
10943 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
10944 PendingExports.push_back(Chain);
10945}
10946
10948
10949/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10950/// entry block, return true. This includes arguments used by switches, since
10951/// the switch may expand into multiple basic blocks.
10952static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
10953 // With FastISel active, we may be splitting blocks, so force creation
10954 // of virtual registers for all non-dead arguments.
10955 if (FastISel)
10956 return A->use_empty();
10957
10958 const BasicBlock &Entry = A->getParent()->front();
10959 for (const User *U : A->users())
10960 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
10961 return false; // Use not in entry block.
10962
10963 return true;
10964}
10965
10967 DenseMap<const Argument *,
10968 std::pair<const AllocaInst *, const StoreInst *>>;
10969
10970/// Scan the entry block of the function in FuncInfo for arguments that look
10971/// like copies into a local alloca. Record any copied arguments in
10972/// ArgCopyElisionCandidates.
10973static void
10975 FunctionLoweringInfo *FuncInfo,
10976 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
10977 // Record the state of every static alloca used in the entry block. Argument
10978 // allocas are all used in the entry block, so we need approximately as many
10979 // entries as we have arguments.
10980 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
10982 unsigned NumArgs = FuncInfo->Fn->arg_size();
10983 StaticAllocas.reserve(NumArgs * 2);
10984
10985 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
10986 if (!V)
10987 return nullptr;
10988 V = V->stripPointerCasts();
10989 const auto *AI = dyn_cast<AllocaInst>(V);
10990 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
10991 return nullptr;
10992 auto Iter = StaticAllocas.insert({AI, Unknown});
10993 return &Iter.first->second;
10994 };
10995
10996 // Look for stores of arguments to static allocas. Look through bitcasts and
10997 // GEPs to handle type coercions, as long as the alloca is fully initialized
10998 // by the store. Any non-store use of an alloca escapes it and any subsequent
10999 // unanalyzed store might write it.
11000 // FIXME: Handle structs initialized with multiple stores.
11001 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11002 // Look for stores, and handle non-store uses conservatively.
11003 const auto *SI = dyn_cast<StoreInst>(&I);
11004 if (!SI) {
11005 // We will look through cast uses, so ignore them completely.
11006 if (I.isCast())
11007 continue;
11008 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11009 // to allocas.
11010 if (I.isDebugOrPseudoInst())
11011 continue;
11012 // This is an unknown instruction. Assume it escapes or writes to all
11013 // static alloca operands.
11014 for (const Use &U : I.operands()) {
11015 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11016 *Info = StaticAllocaInfo::Clobbered;
11017 }
11018 continue;
11019 }
11020
11021 // If the stored value is a static alloca, mark it as escaped.
11022 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11023 *Info = StaticAllocaInfo::Clobbered;
11024
11025 // Check if the destination is a static alloca.
11026 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11027 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11028 if (!Info)
11029 continue;
11030 const AllocaInst *AI = cast<AllocaInst>(Dst);
11031
11032 // Skip allocas that have been initialized or clobbered.
11033 if (*Info != StaticAllocaInfo::Unknown)
11034 continue;
11035
11036 // Check if the stored value is an argument, and that this store fully
11037 // initializes the alloca.
11038 // If the argument type has padding bits we can't directly forward a pointer
11039 // as the upper bits may contain garbage.
11040 // Don't elide copies from the same argument twice.
11041 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11042 const auto *Arg = dyn_cast<Argument>(Val);
11043 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11044 Arg->getType()->isEmptyTy() ||
11045 DL.getTypeStoreSize(Arg->getType()) !=
11046 DL.getTypeAllocSize(AI->getAllocatedType()) ||
11047 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
11048 ArgCopyElisionCandidates.count(Arg)) {
11049 *Info = StaticAllocaInfo::Clobbered;
11050 continue;
11051 }
11052
11053 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11054 << '\n');
11055
11056 // Mark this alloca and store for argument copy elision.
11057 *Info = StaticAllocaInfo::Elidable;
11058 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
11059
11060 // Stop scanning if we've seen all arguments. This will happen early in -O0
11061 // builds, which is useful, because -O0 builds have large entry blocks and
11062 // many allocas.
11063 if (ArgCopyElisionCandidates.size() == NumArgs)
11064 break;
11065 }
11066}
11067
11068/// Try to elide argument copies from memory into a local alloca. Succeeds if
11069/// ArgVal is a load from a suitable fixed stack object.
11072 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11073 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11074 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11075 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11076 // Check if this is a load from a fixed stack object.
11077 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
11078 if (!LNode)
11079 return;
11080 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
11081 if (!FINode)
11082 return;
11083
11084 // Check that the fixed stack object is the right size and alignment.
11085 // Look at the alignment that the user wrote on the alloca instead of looking
11086 // at the stack object.
11087 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
11088 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11089 const AllocaInst *AI = ArgCopyIter->second.first;
11090 int FixedIndex = FINode->getIndex();
11091 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11092 int OldIndex = AllocaIndex;
11093 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11094 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
11095 LLVM_DEBUG(
11096 dbgs() << " argument copy elision failed due to bad fixed stack "
11097 "object size\n");
11098 return;
11099 }
11100 Align RequiredAlignment = AI->getAlign();
11101 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
11102 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11103 "greater than stack argument alignment ("
11104 << DebugStr(RequiredAlignment) << " vs "
11105 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11106 return;
11107 }
11108
11109 // Perform the elision. Delete the old stack object and replace its only use
11110 // in the variable info map. Mark the stack object as mutable.
11111 LLVM_DEBUG({
11112 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11113 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11114 << '\n';
11115 });
11116 MFI.RemoveStackObject(OldIndex);
11117 MFI.setIsImmutableObjectIndex(FixedIndex, false);
11118 AllocaIndex = FixedIndex;
11119 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
11120 for (SDValue ArgVal : ArgVals)
11121 Chains.push_back(ArgVal.getValue(1));
11122
11123 // Avoid emitting code for the store implementing the copy.
11124 const StoreInst *SI = ArgCopyIter->second.second;
11125 ElidedArgCopyInstrs.insert(SI);
11126
11127 // Check for uses of the argument again so that we can avoid exporting ArgVal
11128 // if it is't used by anything other than the store.
11129 for (const Value *U : Arg.users()) {
11130 if (U != SI) {
11131 ArgHasUses = true;
11132 break;
11133 }
11134 }
11135}
11136
11137void SelectionDAGISel::LowerArguments(const Function &F) {
11138 SelectionDAG &DAG = SDB->DAG;
11139 SDLoc dl = SDB->getCurSDLoc();
11140 const DataLayout &DL = DAG.getDataLayout();
11142
11143 // In Naked functions we aren't going to save any registers.
11144 if (F.hasFnAttribute(Attribute::Naked))
11145 return;
11146
11147 if (!FuncInfo->CanLowerReturn) {
11148 // Put in an sret pointer parameter before all the other parameters.
11149 SmallVector<EVT, 1> ValueVTs;
11151 PointerType::get(F.getContext(),
11153 ValueVTs);
11154
11155 // NOTE: Assuming that a pointer will never break down to more than one VT
11156 // or one register.
11158 Flags.setSRet();
11159 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
11160 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
11162 Ins.push_back(RetArg);
11163 }
11164
11165 // Look for stores of arguments to static allocas. Mark such arguments with a
11166 // flag to ask the target to give us the memory location of that argument if
11167 // available.
11168 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11170 ArgCopyElisionCandidates);
11171
11172 // Set up the incoming argument description vector.
11173 for (const Argument &Arg : F.args()) {
11174 unsigned ArgNo = Arg.getArgNo();
11175 SmallVector<EVT, 4> ValueVTs;
11176 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11177 bool isArgValueUsed = !Arg.use_empty();
11178 unsigned PartBase = 0;
11179 Type *FinalType = Arg.getType();
11180 if (Arg.hasAttribute(Attribute::ByVal))
11181 FinalType = Arg.getParamByValType();
11182 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11183 FinalType, F.getCallingConv(), F.isVarArg(), DL);
11184 for (unsigned Value = 0, NumValues = ValueVTs.size();
11185 Value != NumValues; ++Value) {
11186 EVT VT = ValueVTs[Value];
11187 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
11189
11190
11191 if (Arg.getType()->isPointerTy()) {
11192 Flags.setPointer();
11193 Flags.setPointerAddrSpace(
11194 cast<PointerType>(Arg.getType())->getAddressSpace());
11195 }
11196 if (Arg.hasAttribute(Attribute::ZExt))
11197 Flags.setZExt();
11198 if (Arg.hasAttribute(Attribute::SExt))
11199 Flags.setSExt();
11200 if (Arg.hasAttribute(Attribute::InReg)) {
11201 // If we are using vectorcall calling convention, a structure that is
11202 // passed InReg - is surely an HVA
11203 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11204 isa<StructType>(Arg.getType())) {
11205 // The first value of a structure is marked
11206 if (0 == Value)
11207 Flags.setHvaStart();
11208 Flags.setHva();
11209 }
11210 // Set InReg Flag
11211 Flags.setInReg();
11212 }
11213 if (Arg.hasAttribute(Attribute::StructRet))
11214 Flags.setSRet();
11215 if (Arg.hasAttribute(Attribute::SwiftSelf))
11216 Flags.setSwiftSelf();
11217 if (Arg.hasAttribute(Attribute::SwiftAsync))
11218 Flags.setSwiftAsync();
11219 if (Arg.hasAttribute(Attribute::SwiftError))
11220 Flags.setSwiftError();
11221 if (Arg.hasAttribute(Attribute::ByVal))
11222 Flags.setByVal();
11223 if (Arg.hasAttribute(Attribute::ByRef))
11224 Flags.setByRef();
11225 if (Arg.hasAttribute(Attribute::InAlloca)) {
11226 Flags.setInAlloca();
11227 // Set the byval flag for CCAssignFn callbacks that don't know about
11228 // inalloca. This way we can know how many bytes we should've allocated
11229 // and how many bytes a callee cleanup function will pop. If we port
11230 // inalloca to more targets, we'll have to add custom inalloca handling
11231 // in the various CC lowering callbacks.
11232 Flags.setByVal();
11233 }
11234 if (Arg.hasAttribute(Attribute::Preallocated)) {
11235 Flags.setPreallocated();
11236 // Set the byval flag for CCAssignFn callbacks that don't know about
11237 // preallocated. This way we can know how many bytes we should've
11238 // allocated and how many bytes a callee cleanup function will pop. If
11239 // we port preallocated to more targets, we'll have to add custom
11240 // preallocated handling in the various CC lowering callbacks.
11241 Flags.setByVal();
11242 }
11243
11244 // Certain targets (such as MIPS), may have a different ABI alignment
11245 // for a type depending on the context. Give the target a chance to
11246 // specify the alignment it wants.
11247 const Align OriginalAlignment(
11249 Flags.setOrigAlign(OriginalAlignment);
11250
11251 Align MemAlign;
11252 Type *ArgMemTy = nullptr;
11253 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11254 Flags.isByRef()) {
11255 if (!ArgMemTy)
11256 ArgMemTy = Arg.getPointeeInMemoryValueType();
11257
11258 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
11259
11260 // For in-memory arguments, size and alignment should be passed from FE.
11261 // BE will guess if this info is not there but there are cases it cannot
11262 // get right.
11263 if (auto ParamAlign = Arg.getParamStackAlign())
11264 MemAlign = *ParamAlign;
11265 else if ((ParamAlign = Arg.getParamAlign()))
11266 MemAlign = *ParamAlign;
11267 else
11268 MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
11269 if (Flags.isByRef())
11270 Flags.setByRefSize(MemSize);
11271 else
11272 Flags.setByValSize(MemSize);
11273 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
11274 MemAlign = *ParamAlign;
11275 } else {
11276 MemAlign = OriginalAlignment;
11277 }
11278 Flags.setMemAlign(MemAlign);
11279
11280 if (Arg.hasAttribute(Attribute::Nest))
11281 Flags.setNest();
11282 if (NeedsRegBlock)
11283 Flags.setInConsecutiveRegs();
11284 if (ArgCopyElisionCandidates.count(&Arg))
11285 Flags.setCopyElisionCandidate();
11286 if (Arg.hasAttribute(Attribute::Returned))
11287 Flags.setReturned();
11288
11290 *CurDAG->getContext(), F.getCallingConv(), VT);
11291 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
11292 *CurDAG->getContext(), F.getCallingConv(), VT);
11293 for (unsigned i = 0; i != NumRegs; ++i) {
11294 // For scalable vectors, use the minimum size; individual targets
11295 // are responsible for handling scalable vector arguments and
11296 // return values.
11297 ISD::InputArg MyFlags(
11298 Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
11299 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
11300 if (NumRegs > 1 && i == 0)
11301 MyFlags.Flags.setSplit();
11302 // if it isn't first piece, alignment must be 1
11303 else if (i > 0) {
11304 MyFlags.Flags.setOrigAlign(Align(1));
11305 if (i == NumRegs - 1)
11306 MyFlags.Flags.setSplitEnd();
11307 }
11308 Ins.push_back(MyFlags);
11309 }
11310 if (NeedsRegBlock && Value == NumValues - 1)
11311 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11312 PartBase += VT.getStoreSize().getKnownMinValue();
11313 }
11314 }
11315
11316 // Call the target to set up the argument values.
11318 SDValue NewRoot = TLI->LowerFormalArguments(
11319 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
11320
11321 // Verify that the target's LowerFormalArguments behaved as expected.
11322 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
11323 "LowerFormalArguments didn't return a valid chain!");
11324 assert(InVals.size() == Ins.size() &&
11325 "LowerFormalArguments didn't emit the correct number of values!");
11326 LLVM_DEBUG({
11327 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
11328 assert(InVals[i].getNode() &&
11329 "LowerFormalArguments emitted a null value!");
11330 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11331 "LowerFormalArguments emitted a value with the wrong type!");
11332 }
11333 });
11334
11335 // Update the DAG with the new chain value resulting from argument lowering.
11336 DAG.setRoot(NewRoot);
11337
11338 // Set up the argument values.
11339 unsigned i = 0;
11340 if (!FuncInfo->CanLowerReturn) {
11341 // Create a virtual register for the sret pointer, and put in a copy
11342 // from the sret argument into it.
11343 SmallVector<EVT, 1> ValueVTs;
11345 PointerType::get(F.getContext(),
11347 ValueVTs);
11348 MVT VT = ValueVTs[0].getSimpleVT();
11349 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
11350 std::optional<ISD::NodeType> AssertOp;
11351 SDValue ArgValue =
11352 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
11353 F.getCallingConv(), AssertOp);
11354
11355 MachineFunction& MF = SDB->DAG.getMachineFunction();
11357 Register SRetReg =
11358 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
11359 FuncInfo->DemoteRegister = SRetReg;
11360 NewRoot =
11361 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
11362 DAG.setRoot(NewRoot);
11363
11364 // i indexes lowered arguments. Bump it past the hidden sret argument.
11365 ++i;
11366 }
11367
11369 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
11370 for (const Argument &Arg : F.args()) {
11371 SmallVector<SDValue, 4> ArgValues;
11372 SmallVector<EVT, 4> ValueVTs;
11373 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11374 unsigned NumValues = ValueVTs.size();
11375 if (NumValues == 0)
11376 continue;
11377
11378 bool ArgHasUses = !Arg.use_empty();
11379
11380 // Elide the copying store if the target loaded this argument from a
11381 // suitable fixed stack object.
11382 if (Ins[i].Flags.isCopyElisionCandidate()) {
11383 unsigned NumParts = 0;
11384 for (EVT VT : ValueVTs)
11386 F.getCallingConv(), VT);
11387
11388 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
11389 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
11390 ArrayRef(&InVals[i], NumParts), ArgHasUses);
11391 }
11392
11393 // If this argument is unused then remember its value. It is used to generate
11394 // debugging information.
11395 bool isSwiftErrorArg =
11397 Arg.hasAttribute(Attribute::SwiftError);
11398 if (!ArgHasUses && !isSwiftErrorArg) {
11399 SDB->setUnusedArgValue(&Arg, InVals[i]);
11400
11401 // Also remember any frame index for use in FastISel.
11402 if (FrameIndexSDNode *FI =
11403 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
11404 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11405 }
11406
11407 for (unsigned Val = 0; Val != NumValues; ++Val) {
11408 EVT VT = ValueVTs[Val];
11410 F.getCallingConv(), VT);
11411 unsigned NumParts = TLI->getNumRegistersForCallingConv(
11412 *CurDAG->getContext(), F.getCallingConv(), VT);
11413
11414 // Even an apparent 'unused' swifterror argument needs to be returned. So
11415 // we do generate a copy for it that can be used on return from the
11416 // function.
11417 if (ArgHasUses || isSwiftErrorArg) {
11418 std::optional<ISD::NodeType> AssertOp;
11419 if (Arg.hasAttribute(Attribute::SExt))
11420 AssertOp = ISD::AssertSext;
11421 else if (Arg.hasAttribute(Attribute::ZExt))
11422 AssertOp = ISD::AssertZext;
11423
11424 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
11425 PartVT, VT, nullptr, NewRoot,
11426 F.getCallingConv(), AssertOp));
11427 }
11428
11429 i += NumParts;
11430 }
11431
11432 // We don't need to do anything else for unused arguments.
11433 if (ArgValues.empty())
11434 continue;
11435
11436 // Note down frame index.
11437 if (FrameIndexSDNode *FI =
11438 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
11439 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11440
11441 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
11442 SDB->getCurSDLoc());
11443
11444 SDB->setValue(&Arg, Res);
11446 // We want to associate the argument with the frame index, among
11447 // involved operands, that correspond to the lowest address. The
11448 // getCopyFromParts function, called earlier, is swapping the order of
11449 // the operands to BUILD_PAIR depending on endianness. The result of
11450 // that swapping is that the least significant bits of the argument will
11451 // be in the first operand of the BUILD_PAIR node, and the most
11452 // significant bits will be in the second operand.
11453 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
11454 if (LoadSDNode *LNode =
11455 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
11456 if (FrameIndexSDNode *FI =
11457 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
11458 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11459 }
11460
11461 // Analyses past this point are naive and don't expect an assertion.
11462 if (Res.getOpcode() == ISD::AssertZext)
11463 Res = Res.getOperand(0);
11464
11465 // Update the SwiftErrorVRegDefMap.
11466 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
11467 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11470 Reg);
11471 }
11472
11473 // If this argument is live outside of the entry block, insert a copy from
11474 // wherever we got it to the vreg that other BB's will reference it as.
11475 if (Res.getOpcode() == ISD::CopyFromReg) {
11476 // If we can, though, try to skip creating an unnecessary vreg.
11477 // FIXME: This isn't very clean... it would be nice to make this more
11478 // general.
11479 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11480 if (Register::isVirtualRegister(Reg)) {
11481 FuncInfo->ValueMap[&Arg] = Reg;
11482 continue;
11483 }
11484 }
11486 FuncInfo->InitializeRegForValue(&Arg);
11487 SDB->CopyToExportRegsIfNeeded(&Arg);
11488 }
11489 }
11490
11491 if (!Chains.empty()) {
11492 Chains.push_back(NewRoot);
11493 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
11494 }
11495
11496 DAG.setRoot(NewRoot);
11497
11498 assert(i == InVals.size() && "Argument register count mismatch!");
11499
11500 // If any argument copy elisions occurred and we have debug info, update the
11501 // stale frame indices used in the dbg.declare variable info table.
11502 if (!ArgCopyElisionFrameIndexMap.empty()) {
11505 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
11506 if (I != ArgCopyElisionFrameIndexMap.end())
11507 VI.updateStackSlot(I->second);
11508 }
11509 }
11510
11511 // Finally, if the target has anything special to do, allow it to do so.
11513}
11514
11515/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
11516/// ensure constants are generated when needed. Remember the virtual registers
11517/// that need to be added to the Machine PHI nodes as input. We cannot just
11518/// directly add them, because expansion might result in multiple MBB's for one
11519/// BB. As such, the start of the BB might correspond to a different MBB than
11520/// the end.
11521void
11522SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
11524
11526
11527 // Check PHI nodes in successors that expect a value to be available from this
11528 // block.
11529 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
11530 if (!isa<PHINode>(SuccBB->begin())) continue;
11531 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
11532
11533 // If this terminator has multiple identical successors (common for
11534 // switches), only handle each succ once.
11535 if (!SuccsHandled.insert(SuccMBB).second)
11536 continue;
11537
11539
11540 // At this point we know that there is a 1-1 correspondence between LLVM PHI
11541 // nodes and Machine PHI nodes, but the incoming operands have not been
11542 // emitted yet.
11543 for (const PHINode &PN : SuccBB->phis()) {
11544 // Ignore dead phi's.
11545 if (PN.use_empty())
11546 continue;
11547
11548 // Skip empty types
11549 if (PN.getType()->isEmptyTy())
11550 continue;
11551
11552 unsigned Reg;
11553 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
11554
11555 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
11556 unsigned &RegOut = ConstantsOut[C];
11557 if (RegOut == 0) {
11558 RegOut = FuncInfo.CreateRegs(C);
11559 // We need to zero/sign extend ConstantInt phi operands to match
11560 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
11561 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
11562 if (auto *CI = dyn_cast<ConstantInt>(C))
11563 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
11565 CopyValueToVirtualRegister(C, RegOut, ExtendType);
11566 }
11567 Reg = RegOut;
11568 } else {
11570 FuncInfo.ValueMap.find(PHIOp);
11571 if (I != FuncInfo.ValueMap.end())
11572 Reg = I->second;
11573 else {
11574 assert(isa<AllocaInst>(PHIOp) &&
11575 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
11576 "Didn't codegen value into a register!??");
11577 Reg = FuncInfo.CreateRegs(PHIOp);
11578 CopyValueToVirtualRegister(PHIOp, Reg);
11579 }
11580 }
11581
11582 // Remember that this register needs to added to the machine PHI node as
11583 // the input for this MBB.
11584 SmallVector<EVT, 4> ValueVTs;
11585 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
11586 for (EVT VT : ValueVTs) {
11587 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
11588 for (unsigned i = 0; i != NumRegisters; ++i)
11589 FuncInfo.PHINodesToUpdate.push_back(
11590 std::make_pair(&*MBBI++, Reg + i));
11591 Reg += NumRegisters;
11592 }
11593 }
11594 }
11595
11596 ConstantsOut.clear();
11597}
11598
11599MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
11601 if (++I == FuncInfo.MF->end())
11602 return nullptr;
11603 return &*I;
11604}
11605
11606/// During lowering new call nodes can be created (such as memset, etc.).
11607/// Those will become new roots of the current DAG, but complications arise
11608/// when they are tail calls. In such cases, the call lowering will update
11609/// the root, but the builder still needs to know that a tail call has been
11610/// lowered in order to avoid generating an additional return.
11611void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
11612 // If the node is null, we do have a tail call.
11613 if (MaybeTC.getNode() != nullptr)
11614 DAG.setRoot(MaybeTC);
11615 else
11616 HasTailCall = true;
11617}
11618
11619void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
11620 MachineBasicBlock *SwitchMBB,
11621 MachineBasicBlock *DefaultMBB) {
11622 MachineFunction *CurMF = FuncInfo.MF;
11623 MachineBasicBlock *NextMBB = nullptr;
11625 if (++BBI != FuncInfo.MF->end())
11626 NextMBB = &*BBI;
11627
11628 unsigned Size = W.LastCluster - W.FirstCluster + 1;
11629
11631
11632 if (Size == 2 && W.MBB == SwitchMBB) {
11633 // If any two of the cases has the same destination, and if one value
11634 // is the same as the other, but has one bit unset that the other has set,
11635 // use bit manipulation to do two compares at once. For example:
11636 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
11637 // TODO: This could be extended to merge any 2 cases in switches with 3
11638 // cases.
11639 // TODO: Handle cases where W.CaseBB != SwitchBB.
11640 CaseCluster &Small = *W.FirstCluster;
11641 CaseCluster &Big = *W.LastCluster;
11642
11643 if (Small.Low == Small.High && Big.Low == Big.High &&
11644 Small.MBB == Big.MBB) {
11645 const APInt &SmallValue = Small.Low->getValue();
11646 const APInt &BigValue = Big.Low->getValue();
11647
11648 // Check that there is only one bit different.
11649 APInt CommonBit = BigValue ^ SmallValue;
11650 if (CommonBit.isPowerOf2()) {
11651 SDValue CondLHS = getValue(Cond);
11652 EVT VT = CondLHS.getValueType();
11653 SDLoc DL = getCurSDLoc();
11654
11655 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
11656 DAG.getConstant(CommonBit, DL, VT));
11658 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
11659 ISD::SETEQ);
11660
11661 // Update successor info.
11662 // Both Small and Big will jump to Small.BB, so we sum up the
11663 // probabilities.
11664 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
11665 if (BPI)
11666 addSuccessorWithProb(
11667 SwitchMBB, DefaultMBB,
11668 // The default destination is the first successor in IR.
11669 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
11670 else
11671 addSuccessorWithProb(SwitchMBB, DefaultMBB);
11672
11673 // Insert the true branch.
11674 SDValue BrCond =
11675 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
11676 DAG.getBasicBlock(Small.MBB));
11677 // Insert the false branch.
11678 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
11679 DAG.getBasicBlock(DefaultMBB));
11680
11681 DAG.setRoot(BrCond);
11682 return;
11683 }
11684 }
11685 }
11686
11687 if (TM.getOptLevel() != CodeGenOptLevel::None) {
11688 // Here, we order cases by probability so the most likely case will be
11689 // checked first. However, two clusters can have the same probability in
11690 // which case their relative ordering is non-deterministic. So we use Low
11691 // as a tie-breaker as clusters are guaranteed to never overlap.
11692 llvm::sort(W.FirstCluster, W.LastCluster + 1,
11693 [](const CaseCluster &a, const CaseCluster &b) {
11694 return a.Prob != b.Prob ?
11695 a.Prob > b.Prob :
11696 a.Low->getValue().slt(b.Low->getValue());
11697 });
11698
11699 // Rearrange the case blocks so that the last one falls through if possible
11700 // without changing the order of probabilities.
11701 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
11702 --I;
11703 if (I->Prob > W.LastCluster->Prob)
11704 break;
11705 if (I->Kind == CC_Range && I->MBB == NextMBB) {
11706 std::swap(*I, *W.LastCluster);
11707 break;
11708 }
11709 }
11710 }
11711
11712 // Compute total probability.
11713 BranchProbability DefaultProb = W.DefaultProb;
11714 BranchProbability UnhandledProbs = DefaultProb;
11715 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
11716 UnhandledProbs += I->Prob;
11717
11718 MachineBasicBlock *CurMBB = W.MBB;
11719 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
11720 bool FallthroughUnreachable = false;
11721 MachineBasicBlock *Fallthrough;
11722 if (I == W.LastCluster) {
11723 // For the last cluster, fall through to the default destination.
11724 Fallthrough = DefaultMBB;
11725 FallthroughUnreachable = isa<UnreachableInst>(
11726 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
11727 } else {
11728 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
11729 CurMF->insert(BBI, Fallthrough);
11730 // Put Cond in a virtual register to make it available from the new blocks.
11732 }
11733 UnhandledProbs -= I->Prob;
11734
11735 switch (I->Kind) {
11736 case CC_JumpTable: {
11737 // FIXME: Optimize away range check based on pivot comparisons.
11738 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
11739 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
11740
11741 // The jump block hasn't been inserted yet; insert it here.
11742 MachineBasicBlock *JumpMBB = JT->MBB;
11743 CurMF->insert(BBI, JumpMBB);
11744
11745 auto JumpProb = I->Prob;
11746 auto FallthroughProb = UnhandledProbs;
11747
11748 // If the default statement is a target of the jump table, we evenly
11749 // distribute the default probability to successors of CurMBB. Also
11750 // update the probability on the edge from JumpMBB to Fallthrough.
11751 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
11752 SE = JumpMBB->succ_end();
11753 SI != SE; ++SI) {
11754 if (*SI == DefaultMBB) {
11755 JumpProb += DefaultProb / 2;
11756 FallthroughProb -= DefaultProb / 2;
11757 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
11758 JumpMBB->normalizeSuccProbs();
11759 break;
11760 }
11761 }
11762
11763 // If the default clause is unreachable, propagate that knowledge into
11764 // JTH->FallthroughUnreachable which will use it to suppress the range
11765 // check.
11766 //
11767 // However, don't do this if we're doing branch target enforcement,
11768 // because a table branch _without_ a range check can be a tempting JOP
11769 // gadget - out-of-bounds inputs that are impossible in correct
11770 // execution become possible again if an attacker can influence the
11771 // control flow. So if an attacker doesn't already have a BTI bypass
11772 // available, we don't want them to be able to get one out of this
11773 // table branch.
11774 if (FallthroughUnreachable) {
11775 Function &CurFunc = CurMF->getFunction();
11776 bool HasBranchTargetEnforcement = false;
11777 if (CurFunc.hasFnAttribute("branch-target-enforcement")) {
11778 HasBranchTargetEnforcement =
11779 CurFunc.getFnAttribute("branch-target-enforcement")
11780 .getValueAsBool();
11781 } else {
11782 HasBranchTargetEnforcement =
11783 CurMF->getMMI().getModule()->getModuleFlag(
11784 "branch-target-enforcement");
11785 }
11786 if (!HasBranchTargetEnforcement)
11787 JTH->FallthroughUnreachable = true;
11788 }
11789
11790 if (!JTH->FallthroughUnreachable)
11791 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
11792 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
11793 CurMBB->normalizeSuccProbs();
11794
11795 // The jump table header will be inserted in our current block, do the
11796 // range check, and fall through to our fallthrough block.
11797 JTH->HeaderBB = CurMBB;
11798 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
11799
11800 // If we're in the right place, emit the jump table header right now.
11801 if (CurMBB == SwitchMBB) {
11802 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
11803 JTH->Emitted = true;
11804 }
11805 break;
11806 }
11807 case CC_BitTests: {
11808 // FIXME: Optimize away range check based on pivot comparisons.
11809 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
11810
11811 // The bit test blocks haven't been inserted yet; insert them here.
11812 for (BitTestCase &BTC : BTB->Cases)
11813 CurMF->insert(BBI, BTC.ThisBB);
11814
11815 // Fill in fields of the BitTestBlock.
11816 BTB->Parent = CurMBB;
11817 BTB->Default = Fallthrough;
11818
11819 BTB->DefaultProb = UnhandledProbs;
11820 // If the cases in bit test don't form a contiguous range, we evenly
11821 // distribute the probability on the edge to Fallthrough to two
11822 // successors of CurMBB.
11823 if (!BTB->ContiguousRange) {
11824 BTB->Prob += DefaultProb / 2;
11825 BTB->DefaultProb -= DefaultProb / 2;
11826 }
11827
11828 if (FallthroughUnreachable)
11829 BTB->FallthroughUnreachable = true;
11830
11831 // If we're in the right place, emit the bit test header right now.
11832 if (CurMBB == SwitchMBB) {
11833 visitBitTestHeader(*BTB, SwitchMBB);
11834 BTB->Emitted = true;
11835 }
11836 break;
11837 }
11838 case CC_Range: {
11839 const Value *RHS, *LHS, *MHS;
11841 if (I->Low == I->High) {
11842 // Check Cond == I->Low.
11843 CC = ISD::SETEQ;
11844 LHS = Cond;
11845 RHS=I->Low;
11846 MHS = nullptr;
11847 } else {
11848 // Check I->Low <= Cond <= I->High.
11849 CC = ISD::SETLE;
11850 LHS = I->Low;
11851 MHS = Cond;
11852 RHS = I->High;
11853 }
11854
11855 // If Fallthrough is unreachable, fold away the comparison.
11856 if (FallthroughUnreachable)
11857 CC = ISD::SETTRUE;
11858
11859 // The false probability is the sum of all unhandled cases.
11860 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
11861 getCurSDLoc(), I->Prob, UnhandledProbs);
11862
11863 if (CurMBB == SwitchMBB)
11864 visitSwitchCase(CB, SwitchMBB);
11865 else
11866 SL->SwitchCases.push_back(CB);
11867
11868 break;
11869 }
11870 }
11871 CurMBB = Fallthrough;
11872 }
11873}
11874
11875void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
11876 const SwitchWorkListItem &W,
11877 Value *Cond,
11878 MachineBasicBlock *SwitchMBB) {
11879 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
11880 "Clusters not sorted?");
11881 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
11882
11883 auto [LastLeft, FirstRight, LeftProb, RightProb] =
11884 SL->computeSplitWorkItemInfo(W);
11885
11886 // Use the first element on the right as pivot since we will make less-than
11887 // comparisons against it.
11888 CaseClusterIt PivotCluster = FirstRight;
11889 assert(PivotCluster > W.FirstCluster);
11890 assert(PivotCluster <= W.LastCluster);
11891
11892 CaseClusterIt FirstLeft = W.FirstCluster;
11893 CaseClusterIt LastRight = W.LastCluster;
11894
11895 const ConstantInt *Pivot = PivotCluster->Low;
11896
11897 // New blocks will be inserted immediately after the current one.
11899 ++BBI;
11900
11901 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
11902 // we can branch to its destination directly if it's squeezed exactly in
11903 // between the known lower bound and Pivot - 1.
11904 MachineBasicBlock *LeftMBB;
11905 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
11906 FirstLeft->Low == W.GE &&
11907 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
11908 LeftMBB = FirstLeft->MBB;
11909 } else {
11910 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11911 FuncInfo.MF->insert(BBI, LeftMBB);
11912 WorkList.push_back(
11913 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
11914 // Put Cond in a virtual register to make it available from the new blocks.
11916 }
11917
11918 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
11919 // single cluster, RHS.Low == Pivot, and we can branch to its destination
11920 // directly if RHS.High equals the current upper bound.
11921 MachineBasicBlock *RightMBB;
11922 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
11923 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
11924 RightMBB = FirstRight->MBB;
11925 } else {
11926 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
11927 FuncInfo.MF->insert(BBI, RightMBB);
11928 WorkList.push_back(
11929 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
11930 // Put Cond in a virtual register to make it available from the new blocks.
11932 }
11933
11934 // Create the CaseBlock record that will be used to lower the branch.
11935 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
11936 getCurSDLoc(), LeftProb, RightProb);
11937
11938 if (W.MBB == SwitchMBB)
11939 visitSwitchCase(CB, SwitchMBB);
11940 else
11941 SL->SwitchCases.push_back(CB);
11942}
11943
11944// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11945// from the swith statement.
11947 BranchProbability PeeledCaseProb) {
11948 if (PeeledCaseProb == BranchProbability::getOne())
11950 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
11951
11952 uint32_t Numerator = CaseProb.getNumerator();
11953 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
11954 return BranchProbability(Numerator, std::max(Numerator, Denominator));
11955}
11956
11957// Try to peel the top probability case if it exceeds the threshold.
11958// Return current MachineBasicBlock for the switch statement if the peeling
11959// does not occur.
11960// If the peeling is performed, return the newly created MachineBasicBlock
11961// for the peeled switch statement. Also update Clusters to remove the peeled
11962// case. PeeledCaseProb is the BranchProbability for the peeled case.
11963MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
11964 const SwitchInst &SI, CaseClusterVector &Clusters,
11965 BranchProbability &PeeledCaseProb) {
11966 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
11967 // Don't perform if there is only one cluster or optimizing for size.
11968 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
11970 SwitchMBB->getParent()->getFunction().hasMinSize())
11971 return SwitchMBB;
11972
11974 unsigned PeeledCaseIndex = 0;
11975 bool SwitchPeeled = false;
11976 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
11977 CaseCluster &CC = Clusters[Index];
11978 if (CC.Prob < TopCaseProb)
11979 continue;
11980 TopCaseProb = CC.Prob;
11981 PeeledCaseIndex = Index;
11982 SwitchPeeled = true;
11983 }
11984 if (!SwitchPeeled)
11985 return SwitchMBB;
11986
11987 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
11988 << TopCaseProb << "\n");
11989
11990 // Record the MBB for the peeled switch statement.
11991 MachineFunction::iterator BBI(SwitchMBB);
11992 ++BBI;
11993 MachineBasicBlock *PeeledSwitchMBB =
11995 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
11996
11997 ExportFromCurrentBlock(SI.getCondition());
11998 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
11999 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
12000 nullptr, nullptr, TopCaseProb.getCompl()};
12001 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
12002
12003 Clusters.erase(PeeledCaseIt);
12004 for (CaseCluster &CC : Clusters) {
12005 LLVM_DEBUG(
12006 dbgs() << "Scale the probablity for one cluster, before scaling: "
12007 << CC.Prob << "\n");
12008 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
12009 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12010 }
12011 PeeledCaseProb = TopCaseProb;
12012 return PeeledSwitchMBB;
12013}
12014
12015void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12016 // Extract cases from the switch.
12018 CaseClusterVector Clusters;
12019 Clusters.reserve(SI.getNumCases());
12020 for (auto I : SI.cases()) {
12021 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
12022 const ConstantInt *CaseVal = I.getCaseValue();
12023 BranchProbability Prob =
12024 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
12025 : BranchProbability(1, SI.getNumCases() + 1);
12026 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
12027 }
12028
12029 MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
12030
12031 // Cluster adjacent cases with the same destination. We do this at all
12032 // optimization levels because it's cheap to do and will make codegen faster
12033 // if there are many clusters.
12034 sortAndRangeify(Clusters);
12035
12036 // The branch probablity of the peeled case.
12038 MachineBasicBlock *PeeledSwitchMBB =
12039 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12040
12041 // If there is only the default destination, jump there directly.
12042 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12043 if (Clusters.empty()) {
12044 assert(PeeledSwitchMBB == SwitchMBB);
12045 SwitchMBB->addSuccessor(DefaultMBB);
12046 if (DefaultMBB != NextBlock(SwitchMBB)) {
12047 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
12048 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
12049 }
12050 return;
12051 }
12052
12053 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
12054 DAG.getBFI());
12055 SL->findBitTestClusters(Clusters, &SI);
12056
12057 LLVM_DEBUG({
12058 dbgs() << "Case clusters: ";
12059 for (const CaseCluster &C : Clusters) {
12060 if (C.Kind == CC_JumpTable)
12061 dbgs() << "JT:";
12062 if (C.Kind == CC_BitTests)
12063 dbgs() << "BT:";
12064
12065 C.Low->getValue().print(dbgs(), true);
12066 if (C.Low != C.High) {
12067 dbgs() << '-';
12068 C.High->getValue().print(dbgs(), true);
12069 }
12070 dbgs() << ' ';
12071 }
12072 dbgs() << '\n';
12073 });
12074
12075 assert(!Clusters.empty());
12076 SwitchWorkList WorkList;
12077 CaseClusterIt First = Clusters.begin();
12078 CaseClusterIt Last = Clusters.end() - 1;
12079 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
12080 // Scale the branchprobability for DefaultMBB if the peel occurs and
12081 // DefaultMBB is not replaced.
12082 if (PeeledCaseProb != BranchProbability::getZero() &&
12083 DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
12084 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
12085 WorkList.push_back(
12086 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
12087
12088 while (!WorkList.empty()) {
12089 SwitchWorkListItem W = WorkList.pop_back_val();
12090 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12091
12092 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12093 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12094 // For optimized builds, lower large range as a balanced binary tree.
12095 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
12096 continue;
12097 }
12098
12099 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
12100 }
12101}
12102
12103void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12105 auto DL = getCurSDLoc();
12106 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12107 setValue(&I, DAG.getStepVector(DL, ResultVT));
12108}
12109
12110void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12112 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12113
12114 SDLoc DL = getCurSDLoc();
12115 SDValue V = getValue(I.getOperand(0));
12116 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12117
12118 if (VT.isScalableVector()) {
12120 return;
12121 }
12122
12123 // Use VECTOR_SHUFFLE for the fixed-length vector
12124 // to maintain existing behavior.
12126 unsigned NumElts = VT.getVectorMinNumElements();
12127 for (unsigned i = 0; i != NumElts; ++i)
12128 Mask.push_back(NumElts - 1 - i);
12129
12130 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
12131}
12132
12133void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
12134 auto DL = getCurSDLoc();
12135 SDValue InVec = getValue(I.getOperand(0));
12136 EVT OutVT =
12138
12139 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12140
12141 // ISD Node needs the input vectors split into two equal parts
12142 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12144 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12145 DAG.getVectorIdxConstant(OutNumElts, DL));
12146
12147 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12148 // legalisation and combines.
12149 if (OutVT.isFixedLengthVector()) {
12150 SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12151 createStrideMask(0, 2, OutNumElts));
12152 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12153 createStrideMask(1, 2, OutNumElts));
12154 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
12155 setValue(&I, Res);
12156 return;
12157 }
12158
12160 DAG.getVTList(OutVT, OutVT), Lo, Hi);
12161 setValue(&I, Res);
12162}
12163
12164void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
12165 auto DL = getCurSDLoc();
12166 EVT InVT = getValue(I.getOperand(0)).getValueType();
12167 SDValue InVec0 = getValue(I.getOperand(0));
12168 SDValue InVec1 = getValue(I.getOperand(1));
12170 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12171
12172 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12173 // legalisation and combines.
12174 if (OutVT.isFixedLengthVector()) {
12175 unsigned NumElts = InVT.getVectorMinNumElements();
12176 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
12177 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
12178 createInterleaveMask(NumElts, 2)));
12179 return;
12180 }
12181
12183 DAG.getVTList(InVT, InVT), InVec0, InVec1);
12184 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
12185 Res.getValue(1));
12186 setValue(&I, Res);
12187}
12188
12189void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12190 SmallVector<EVT, 4> ValueVTs;
12192 ValueVTs);
12193 unsigned NumValues = ValueVTs.size();
12194 if (NumValues == 0) return;
12195
12196 SmallVector<SDValue, 4> Values(NumValues);
12197 SDValue Op = getValue(I.getOperand(0));
12198
12199 for (unsigned i = 0; i != NumValues; ++i)
12200 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
12201 SDValue(Op.getNode(), Op.getResNo() + i));
12202
12204 DAG.getVTList(ValueVTs), Values));
12205}
12206
12207void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12209 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12210
12211 SDLoc DL = getCurSDLoc();
12212 SDValue V1 = getValue(I.getOperand(0));
12213 SDValue V2 = getValue(I.getOperand(1));
12214 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
12215
12216 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
12217 if (VT.isScalableVector()) {
12218 MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
12219 setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
12220 DAG.getConstant(Imm, DL, IdxVT)));
12221 return;
12222 }
12223
12224 unsigned NumElts = VT.getVectorNumElements();
12225
12226 uint64_t Idx = (NumElts + Imm) % NumElts;
12227
12228 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12230 for (unsigned i = 0; i < NumElts; ++i)
12231 Mask.push_back(Idx + i);
12232 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
12233}
12234
12235// Consider the following MIR after SelectionDAG, which produces output in
12236// phyregs in the first case or virtregs in the second case.
12237//
12238// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12239// %5:gr32 = COPY $ebx
12240// %6:gr32 = COPY $edx
12241// %1:gr32 = COPY %6:gr32
12242// %0:gr32 = COPY %5:gr32
12243//
12244// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12245// %1:gr32 = COPY %6:gr32
12246// %0:gr32 = COPY %5:gr32
12247//
12248// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12249// Given %1, we'd like to return $edx in the first case and %6 in the second.
12250//
12251// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12252// to a single virtreg (such as %0). The remaining outputs monotonically
12253// increase in virtreg number from there. If a callbr has no outputs, then it
12254// should not have a corresponding callbr landingpad; in fact, the callbr
12255// landingpad would not even be able to refer to such a callbr.
12257 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
12258 // There is definitely at least one copy.
12259 assert(MI->getOpcode() == TargetOpcode::COPY &&
12260 "start of copy chain MUST be COPY");
12261 Reg = MI->getOperand(1).getReg();
12262 MI = MRI.def_begin(Reg)->getParent();
12263 // There may be an optional second copy.
12264 if (MI->getOpcode() == TargetOpcode::COPY) {
12265 assert(Reg.isVirtual() && "expected COPY of virtual register");
12266 Reg = MI->getOperand(1).getReg();
12267 assert(Reg.isPhysical() && "expected COPY of physical register");
12268 MI = MRI.def_begin(Reg)->getParent();
12269 }
12270 // The start of the chain must be an INLINEASM_BR.
12271 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12272 "end of copy chain MUST be INLINEASM_BR");
12273 return Reg;
12274}
12275
12276// We must do this walk rather than the simpler
12277// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
12278// otherwise we will end up with copies of virtregs only valid along direct
12279// edges.
12280void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
12281 SmallVector<EVT, 8> ResultVTs;
12282 SmallVector<SDValue, 8> ResultValues;
12283 const auto *CBR =
12284 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
12285
12289
12290 unsigned InitialDef = FuncInfo.ValueMap[CBR];
12291 SDValue Chain = DAG.getRoot();
12292
12293 // Re-parse the asm constraints string.
12294 TargetLowering::AsmOperandInfoVector TargetConstraints =
12295 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
12296 for (auto &T : TargetConstraints) {
12297 SDISelAsmOperandInfo OpInfo(T);
12298 if (OpInfo.Type != InlineAsm::isOutput)
12299 continue;
12300
12301 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
12302 // individual constraint.
12303 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
12304
12305 switch (OpInfo.ConstraintType) {
12308 // Fill in OpInfo.AssignedRegs.Regs.
12309 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
12310
12311 // getRegistersForValue may produce 1 to many registers based on whether
12312 // the OpInfo.ConstraintVT is legal on the target or not.
12313 for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) {
12314 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
12315 if (Register::isPhysicalRegister(OriginalDef))
12316 FuncInfo.MBB->addLiveIn(OriginalDef);
12317 // Update the assigned registers to use the original defs.
12318 OpInfo.AssignedRegs.Regs[i] = OriginalDef;
12319 }
12320
12321 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
12322 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
12323 ResultValues.push_back(V);
12324 ResultVTs.push_back(OpInfo.ConstraintVT);
12325 break;
12326 }
12328 SDValue Flag;
12329 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
12330 OpInfo, DAG);
12331 ++InitialDef;
12332 ResultValues.push_back(V);
12333 ResultVTs.push_back(OpInfo.ConstraintVT);
12334 break;
12335 }
12336 default:
12337 break;
12338 }
12339 }
12341 DAG.getVTList(ResultVTs), ResultValues);
12342 setValue(&I, V);
12343}
unsigned const MachineRegisterInfo * MRI
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition: FastISel.cpp:934
#define Check(C,...)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
This file provides utility analysis objects describing memory locations.
This file contains the declarations for metadata subclasses.
#define T1
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Module.h This file contains the declarations for the Module class.
uint64_t High
LLVMContext & Context
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< unsigned, TypeSize > > &Regs, const SDValue &N)
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, ISD::MemIndexType &IndexType, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static void findWasmUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1010
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
an instruction to allocate memory on the stack
Definition: Instructions.h:59
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:125
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition: Function.cpp:330
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:46
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
This class represents the atomic memcpy intrinsic i.e.
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
@ Add
*p = old + v
Definition: Instructions.h:764
@ FAdd
*p = old + v
Definition: Instructions.h:785
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:778
@ Or
*p = old | v
Definition: Instructions.h:772
@ Sub
*p = old - v
Definition: Instructions.h:766
@ And
*p = old & v
Definition: Instructions.h:768
@ Xor
*p = old ^ v
Definition: Instructions.h:774
@ FSub
*p = old - v
Definition: Instructions.h:788
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:800
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:776
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:782
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:796
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:780
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:792
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:804
@ Nand
*p = ~(old & v)
Definition: Instructions.h:770
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
bool getValueAsBool() const
Return the attribute's value as a boolean.
Definition: Attributes.cpp:335
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:347
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
Definition: BasicBlock.cpp:551
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:366
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:220
const Instruction & back() const
Definition: BasicBlock.h:454
This class represents a no-op cast from one type to another.
bool test(unsigned Idx) const
Definition: BitVector.h:461
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
BitVector & set()
Definition: BitVector.h:351
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
The address of a basic block.
Definition: Constants.h:888
Conditional or Unconditional Branch instruction.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static uint32_t getDenominator()
static BranchProbability getOne()
uint32_t getNumerator() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1455
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2357
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1761
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1623
bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2333
Value * getCalledOperand() const
Definition: InstrTypes.h:1696
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1648
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1629
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2241
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1561
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
unsigned arg_size() const
Definition: InstrTypes.h:1646
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1780
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:955
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition: Constants.h:582
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1016
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2140
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:267
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:204
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:856
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:144
This class represents a range of values.
Definition: ConstantRange.h:47
APInt getUnsignedMin() const
Return the smallest unsigned value contained in the ConstantRange.
bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
bool isEmptySet() const
Return true if this set contains no members.
bool isUpperWrapped() const
Return true if the exclusive upper bound wraps around the unsigned domain.
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition: Constants.h:506
This is an important base class in LLVM.
Definition: Constant.h:41
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Debug location.
Base class for variables.
std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
Records a position in IR for a source label (DILabel).
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
LocationType getType() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
bool isBigEndian() const
Definition: DataLayout.h:239
unsigned getAllocaAddrSpace() const
Definition: DataLayout.h:276
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
Definition: DataLayout.h:420
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:874
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
iterator_range< location_op_iterator > getValues() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
A debug info location.
Definition: DebugLoc.h:33
DILocation * getInlinedAt() const
Definition: DebugLoc.cpp:39
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:296
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition: TypeSize.h:302
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:307
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
bool allowReassoc() const
Flag queries.
Definition: FMF.h:65
An instruction for ordering other memory operations.
Definition: Instructions.h:460
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
Register CreateRegs(const Value *V)
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
BitVector DescribedArgs
Bitvector with a bit set if corresponding argument is described in ArgDbgValues.
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
SmallPtrSet< const DPValue *, 8 > PreprocessedDPVDeclares
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
Register InitializeRegForValue(const Value *V)
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineBasicBlock * MBB
MBB - The current block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
MachineRegisterInfo * RegInfo
Register CreateReg(MVT VT, bool isDivergent=false)
CreateReg - Allocate a single virtual register for the given type.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
Register getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:142
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
Type * getReturnType() const
Definition: DerivedTypes.h:124
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition: Function.h:782
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:695
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:230
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:677
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:330
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:262
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1874
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:338
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:235
size_t arg_size() const
Definition: Function.h:846
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition: Function.h:213
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:669
Garbage collection metadata for a single function.
Definition: GCMetadata.h:78
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Definition: GCMetadata.h:118
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:973
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:566
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
This instruction inserts a struct field of array element value into an aggregate value.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:453
const BasicBlock * getParent() const
Definition: Instruction.h:151
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1704
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Definition: DerivedTypes.h:51
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:184
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:321
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
Machine Value Type.
uint64_t getScalarSizeInBits() const
@ INVALID_SIMPLE_VALUE_TYPE
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void setIsEHCatchretTarget(bool V=true)
Indicates if this is a target block of a catchret.
void setIsCleanupFuncletEntry(bool V=true)
Indicates if this is the entry block of a cleanup funclet.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasPatchPoint(bool s=true)
void setHasStackMap(bool s=true)
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
Description of the location of a variable whose Address is valid and unchanging during function execu...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
void setCallsUnwindInit(bool b)
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void setHasEHCatchret(bool V)
void setCallsEHReturn(bool b)
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
auto getInStackSlotVariableDbgInfo()
Returns the collection of variables for which we have debug info and that have been assigned a stack ...
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
const MCContext & getContext() const
const Module * getModule() const
void setCurrentCallSite(unsigned Site)
Set the call site currently being processed.
unsigned getCurrentCallSite()
Get the call site currently being processed, if any.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
ArrayRef< std::pair< MCRegister, Register > > liveins() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool contains(const KeyT &Key) const
Definition: MapVector.h:163
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition: MapVector.h:118
Representation for a specific memory location.
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:176
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Definition: Module.cpp:331
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition: Operator.h:75
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1827
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed.
Definition: Operator.h:150
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
void CopyValueToVirtualRegister(const Value *V, unsigned Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, unsigned Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const BranchInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void init(GCFunctionInfo *gfi, AAResults *AA, AssumptionCache *AC, const TargetLibraryInfo *li)
DenseMap< const Constant *, unsigned > ConstantsOut
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineFunction * MF
virtual void emitFunctionEntryCode()
SwiftErrorValueTracking * SwiftError
std::unique_ptr< SelectionDAGBuilder > SDB
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
Help to insert SDNodeFlags automatically in transforming.
Definition: SelectionDAG.h:361
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:722
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:954
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:474
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
BlockFrequencyInfo * getBFI() const
Definition: SelectionDAG.h:488
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
bool shouldOptForSize() const
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
ProfileSummaryInfo * getPSI() const
Definition: SelectionDAG.h:487
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:727
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:480
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:862
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:773
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:708
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:799
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
Definition: SelectionDAG.h:484
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
void addPCSections(const SDNode *Node, MDNode *MD)
Set PCSections to be associated with Node.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:878
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:981
void resize(size_type N)
Definition: SmallVector.h:651
void push_back(const T &Elt)
Definition: SmallVector.h:426
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
void clear()
Clear the memory usage of this object.
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
TargetIntrinsicInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *) const
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
std::vector< ArgListEntry > ArgListTy
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual const char * getClearCacheBuiltinName() const
Return the builtin name for the __builtin___clear_cache intrinsic Default is to invoke the clear cach...
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const
Return a TargetTransformInfo for a given function.
TargetOptions Options
CodeModel::Model getCodeModel() const
Returns the code model.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
unsigned getID() const
Return the register class ID number.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:361
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:750
bool isWasm() const
Tests whether the target is wasm (32- and 64-bit).
Definition: Triple.h:1005
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
Definition: Triple.h:895
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:255
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition: Type.h:216
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
Base class of all SIMD vector types.
Definition: DerivedTypes.h:403
Type * getElementType() const
Definition: DerivedTypes.h:436
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:86
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition: CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
Definition: CallingConv.h:245
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
Definition: CallingConv.h:163
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1126
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1122
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ CONVERGENCECTRL_ANCHOR
Definition: ISDOpcodes.h:1390
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1276
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:998
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1370
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:147
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1269
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1155
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1271
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1241
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1272
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1002
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1021
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:151
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1254
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1233
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1025
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1369
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FPTRUNC_ROUND
Definition: ISDOpcodes.h:480
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1267
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1268
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1199
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1274
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:135
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1108
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:885
@ CONVERGENCECTRL_GLUE
Definition: ISDOpcodes.h:1396
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ PREALLOCATED_SETUP
Definition: ISDOpcodes.h:1160
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1188
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:101
@ CONVERGENCECTRL_ENTRY
Definition: ISDOpcodes.h:1391
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:1047
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1221
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1275
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1270
@ PREALLOCATED_ARG
Definition: ISDOpcodes.h:1163
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:1052
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1056
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
Definition: ISDOpcodes.h:586
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:507
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1237
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1371
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1151
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1277
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1225
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1364
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:880
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:1117
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1016
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:993
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1265
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:118
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1211
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1329
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1273
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:114
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1215
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1041
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition: ISDOpcodes.h:591
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1372
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
Definition: ISDOpcodes.h:1174
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
Definition: ISDOpcodes.h:1097
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition: ISDOpcodes.h:129
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1279
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1263
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1264
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1182
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1208
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition: ISDOpcodes.h:106
@ CONVERGENCECTRL_LOOP
Definition: ISDOpcodes.h:1392
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1094
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:141
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1070
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:1113
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1278
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
Definition: ISDOpcodes.h:580
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1320
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1205
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1485
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:821
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
VScaleVal_match m_VScale()
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1522
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Reg
All possible values of the reg field in the ModR/M byte.
@ ReallyHidden
Definition: CommandLine.h:139
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:470
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:146
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebStrict
This corresponds to "fpexcept.strict".
Definition: FPEnv.h:41
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition: FPEnv.h:40
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
constexpr float log2ef
Definition: MathExtras.h:50
constexpr double e
Definition: MathExtras.h:31
constexpr float ln2f
Definition: MathExtras.h:48
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
@ Offset
Definition: DWP.cpp:456
@ Length
Definition: DWP.cpp:456
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1689
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
@ Done
Definition: Threading.h:61
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2082
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
gep_type_iterator gep_type_end(const User *GEP)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:342
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition: STLExtras.h:885
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition: STLExtras.h:322
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition: Analysis.cpp:199
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition: Local.cpp:2523
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2322
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
DWARFExpression::Operation Op
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition: Analysis.cpp:221
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2060
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:535
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2048
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:428
unsigned succ_size(const MachineBasicBlock *BB)
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:33
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition: bit.h:327
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
#define NC
Definition: regutils.h:42
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:249
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
uint64_t getScalarStoreSize() const
Definition: ValueTypes.h:387
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:349
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:366
bool isScalableVT() const
Return true if the type is a scalable type.
Definition: ValueTypes.h:183
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:156
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:101
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition: InlineAsm.h:126
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:244
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< unsigned, 4 > Regs
This list holds the registers assigned to the values.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< std::pair< unsigned, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
bool hasAllowReassociation() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
A cluster of case labels.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)