LLVM 24.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsBPF.h"
82#include "llvm/IR/IntrinsicsWebAssembly.h"
83#include "llvm/IR/LLVMContext.h"
86#include "llvm/IR/Metadata.h"
87#include "llvm/IR/Module.h"
88#include "llvm/IR/Operator.h"
91#include "llvm/IR/Statepoint.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCAsmInfo.h"
96#include "llvm/MC/MCContext.h"
101#include "llvm/Support/Debug.h"
109#include <cstddef>
110#include <limits>
111#include <optional>
112#include <tuple>
113
114using namespace llvm;
115using namespace PatternMatch;
116using namespace SwitchCG;
117
118#define DEBUG_TYPE "isel"
119
120/// LimitFloatPrecision - Generate low-precision inline sequences for
121/// some float libcalls (6, 8 or 12 bits).
122static unsigned LimitFloatPrecision;
123
124static cl::opt<bool>
125 InsertAssertAlign("insert-assert-align", cl::init(true),
126 cl::desc("Insert the experimental `assertalign` node."),
128
130 LimitFPPrecision("limit-float-precision",
131 cl::desc("Generate low-precision inline sequences "
132 "for some float libcalls"),
134 cl::init(0));
135
137 "switch-peel-threshold", cl::Hidden, cl::init(66),
138 cl::desc("Set the case probability threshold for peeling the case from a "
139 "switch statement. A value greater than 100 will void this "
140 "optimization"));
141
142// Limit the width of DAG chains. This is important in general to prevent
143// DAG-based analysis from blowing up. For example, alias analysis and
144// load clustering may not complete in reasonable time. It is difficult to
145// recognize and avoid this situation within each individual analysis, and
146// future analyses are likely to have the same behavior. Limiting DAG width is
147// the safe approach and will be especially important with global DAGs.
148//
149// MaxParallelChains default is arbitrarily high to avoid affecting
150// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
151// sequence over this should have been converted to llvm.memcpy by the
152// frontend. It is easy to induce this behavior with .ll code such as:
153// %buffer = alloca [4096 x i8]
154// %data = load [4096 x i8]* %argPtr
155// store [4096 x i8] %data, [4096 x i8]* %buffer
156static const unsigned MaxParallelChains = 64;
157
159 const SDValue *Parts, unsigned NumParts,
160 MVT PartVT, EVT ValueVT, const Value *V,
161 SDValue InChain,
162 std::optional<CallingConv::ID> CC);
163
164/// getCopyFromParts - Create a value that contains the specified legal parts
165/// combined into the value they represent. If the parts combine to a type
166/// larger than ValueVT then AssertOp can be used to specify whether the extra
167/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
168/// (ISD::AssertSext).
169static SDValue
170getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
171 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
172 SDValue InChain,
173 std::optional<CallingConv::ID> CC = std::nullopt,
174 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
175 // Let the target assemble the parts if it wants to
177 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
178 PartVT, ValueVT, CC))
179 return Val;
180
181 if (ValueVT.isVector())
182 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
183 InChain, CC);
184
185 assert(NumParts > 0 && "No parts to assemble!");
186 SDValue Val = Parts[0];
187
188 if (NumParts > 1) {
189 // Assemble the value from multiple parts.
190 if (ValueVT.isInteger()) {
191 unsigned PartBits = PartVT.getSizeInBits();
192 unsigned ValueBits = ValueVT.getSizeInBits();
193
194 // Assemble the power of 2 part.
195 unsigned RoundParts = llvm::bit_floor(NumParts);
196 unsigned RoundBits = PartBits * RoundParts;
197 EVT RoundVT = RoundBits == ValueBits ?
198 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
199 SDValue Lo, Hi;
200
201 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
202
203 if (RoundParts > 2) {
204 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
205 InChain);
206 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
207 PartVT, HalfVT, V, InChain);
208 } else {
209 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
210 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
211 }
212
213 if (DAG.getDataLayout().isBigEndian())
214 std::swap(Lo, Hi);
215
216 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
217
218 if (RoundParts < NumParts) {
219 // Assemble the trailing non-power-of-2 part.
220 unsigned OddParts = NumParts - RoundParts;
221 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
222 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
223 OddVT, V, InChain, CC);
224
225 // Combine the round and odd parts.
226 Lo = Val;
227 if (DAG.getDataLayout().isBigEndian())
228 std::swap(Lo, Hi);
229 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
230 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
231 Hi = DAG.getNode(
232 ISD::SHL, DL, TotalVT, Hi,
233 DAG.getShiftAmountConstant(Lo.getValueSizeInBits(), TotalVT, DL));
234 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
235 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
236 }
237 } else if (PartVT.isFloatingPoint()) {
238 // FP split into multiple FP parts (for ppcf128)
239 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
240 "Unexpected split");
241 SDValue Lo, Hi;
242 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
243 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
244 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
245 std::swap(Lo, Hi);
246 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
247 } else {
248 // FP split into integer parts (soft fp)
249 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
250 !PartVT.isVector() && "Unexpected split");
251 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
252 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
253 InChain, CC);
254 }
255 }
256
257 // There is now one part, held in Val. Correct it to match ValueVT.
258 // PartEVT is the type of the register class that holds the value.
259 // ValueVT is the type of the inline asm operation.
260 EVT PartEVT = Val.getValueType();
261
262 if (PartEVT == ValueVT)
263 return Val;
264
265 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
266 ValueVT.bitsLT(PartEVT)) {
267 // For an FP value in an integer part, we need to truncate to the right
268 // width first.
269 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
270 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
271 }
272
273 // Handle types that have the same size.
274 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
275 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
276
277 // Handle types with different sizes.
278 if (PartEVT.isInteger() && ValueVT.isInteger()) {
279 if (ValueVT.bitsLT(PartEVT)) {
280 // For a truncate, see if we have any information to
281 // indicate whether the truncated bits will always be
282 // zero or sign-extension.
283 if (AssertOp)
284 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
285 DAG.getValueType(ValueVT));
286 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
287 }
288 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
289 }
290
291 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
292 // FP_ROUND's are always exact here.
293 if (ValueVT.bitsLT(Val.getValueType())) {
294
295 SDValue NoChange =
296 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
297
298 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
299 llvm::Attribute::StrictFP)) {
300 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
301 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
302 NoChange);
303 }
304
305 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
306 }
307
308 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
309 }
310
311 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
312 // then truncating.
313 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
314 ValueVT.bitsLT(PartEVT)) {
315 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
316 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
317 }
318
319 report_fatal_error("Unknown mismatch in getCopyFromParts!");
320}
321
323 const Twine &ErrMsg) {
325 if (!I)
326 return Ctx.emitError(ErrMsg);
327
328 if (const CallInst *CI = dyn_cast<CallInst>(I))
329 if (CI->isInlineAsm()) {
330 return Ctx.diagnose(DiagnosticInfoInlineAsm(
331 *CI, ErrMsg + ", possible invalid constraint for vector type"));
332 }
333
334 return Ctx.emitError(I, ErrMsg);
335}
336
337/// Emit a fatal error if the broken down registers don't match part type
338/// or count.
339static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts,
340 MVT RegisterVT, MVT PartVT) {
341 if (NumRegs == NumParts && RegisterVT == PartVT)
342 return;
343
345 Twine("Part count doesn't match vector breakdown! ") + Twine(NumRegs) +
346 " registers, " + Twine(NumParts) + " parts, " + RegisterVT.getString() +
347 " RegisterVT, " + PartVT.getString() + " PartVT");
348}
349
350/// getCopyFromPartsVector - Create a value that contains the specified legal
351/// parts combined into the value they represent. If the parts combine to a
352/// type larger than ValueVT then AssertOp can be used to specify whether the
353/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
354/// ValueVT (ISD::AssertSext).
356 const SDValue *Parts, unsigned NumParts,
357 MVT PartVT, EVT ValueVT, const Value *V,
358 SDValue InChain,
359 std::optional<CallingConv::ID> CallConv) {
360 assert(ValueVT.isVector() && "Not a vector value");
361 assert(NumParts > 0 && "No parts to assemble!");
362 const bool IsABIRegCopy = CallConv.has_value();
363
365 SDValue Val = Parts[0];
366
367 // Handle a multi-element vector.
368 if (NumParts > 1) {
369 EVT IntermediateVT;
370 MVT RegisterVT;
371 unsigned NumIntermediates;
372 unsigned NumRegs;
373
374 if (IsABIRegCopy) {
375 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
376 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
377 NumIntermediates, RegisterVT);
378 } else {
379 NumRegs =
380 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
381 NumIntermediates, RegisterVT);
382 }
383
384 ensureMatchedVecRegParts(NumRegs, NumParts, RegisterVT, PartVT);
385 assert(RegisterVT.getSizeInBits() ==
386 Parts[0].getSimpleValueType().getSizeInBits() &&
387 "Part type sizes don't match!");
388
389 // Assemble the parts into intermediate operands.
390 SmallVector<SDValue, 8> Ops(NumIntermediates);
391 if (NumIntermediates == NumParts) {
392 // If the register was not expanded, truncate or copy the value,
393 // as appropriate.
394 for (unsigned i = 0; i != NumParts; ++i)
395 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
396 V, InChain, CallConv);
397 } else if (NumParts > 0) {
398 // If the intermediate type was expanded, build the intermediate
399 // operands from the parts.
400 assert(NumParts % NumIntermediates == 0 &&
401 "Must expand into a divisible number of parts!");
402 unsigned Factor = NumParts / NumIntermediates;
403 for (unsigned i = 0; i != NumIntermediates; ++i)
404 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
405 IntermediateVT, V, InChain, CallConv);
406 }
407
408 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
409 // intermediate operands.
410 EVT BuiltVectorTy =
411 IntermediateVT.isVector()
413 *DAG.getContext(), IntermediateVT.getScalarType(),
414 IntermediateVT.getVectorElementCount() * NumParts)
416 IntermediateVT.getScalarType(),
417 NumIntermediates);
418 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
420 DL, BuiltVectorTy, Ops);
421 }
422
423 // There is now one part, held in Val. Correct it to match ValueVT.
424 EVT PartEVT = Val.getValueType();
425
426 if (PartEVT == ValueVT)
427 return Val;
428
429 if (PartEVT.isVector()) {
430 // Vector/Vector bitcast.
431 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
432 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
433
434 // If the parts vector has more elements than the value vector, then we
435 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
436 // Extract the elements we want.
437 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
440 (PartEVT.getVectorElementCount().isScalable() ==
441 ValueVT.getVectorElementCount().isScalable()) &&
442 "Cannot narrow, it would be a lossy transformation");
443 PartEVT =
445 ValueVT.getVectorElementCount());
446 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
447 DAG.getVectorIdxConstant(0, DL));
448 if (PartEVT == ValueVT)
449 return Val;
450 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
451 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
452
453 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
454 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
455 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
456 }
457
458 // Promoted vector extract
459 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
460 }
461
462 // Trivial bitcast if the types are the same size and the destination
463 // vector type is legal.
464 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
465 TLI.isTypeLegal(ValueVT))
466 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
467
468 if (ValueVT.getVectorNumElements() != 1) {
469 // Certain ABIs require that vectors are passed as integers. For vectors
470 // are the same size, this is an obvious bitcast.
471 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
472 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
473 } else if (ValueVT.bitsLT(PartEVT)) {
474 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
475 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
476 // Drop the extra bits.
477 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
478 return DAG.getBitcast(ValueVT, Val);
479 }
480
482 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
483 return DAG.getUNDEF(ValueVT);
484 }
485
486 // Handle cases such as i8 -> <1 x i1>
487 EVT ValueSVT = ValueVT.getVectorElementType();
488 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
489 unsigned ValueSize = ValueSVT.getSizeInBits();
490 if (ValueSize == PartEVT.getSizeInBits()) {
491 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
492 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
493 // It's possible a scalar floating point type gets softened to integer and
494 // then promoted to a larger integer. If PartEVT is the larger integer
495 // we need to truncate it and then bitcast to the FP type.
496 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
497 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
498 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
499 Val = DAG.getBitcast(ValueSVT, Val);
500 } else {
501 Val = ValueVT.isFloatingPoint()
502 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
503 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
504 }
505 }
506
507 return DAG.getBuildVector(ValueVT, DL, Val);
508}
509
510static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
511 SDValue Val, SDValue *Parts, unsigned NumParts,
512 MVT PartVT, const Value *V,
513 std::optional<CallingConv::ID> CallConv);
514
515/// getCopyToParts - Create a series of nodes that contain the specified value
516/// split into legal parts. If the parts contain more bits than Val, then, for
517/// integers, ExtendKind can be used to specify how to generate the extra bits.
518static void
520 unsigned NumParts, MVT PartVT, const Value *V,
521 std::optional<CallingConv::ID> CallConv = std::nullopt,
523 // Let the target split the parts if it wants to
525 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
526 CallConv))
527 return;
528 EVT ValueVT = Val.getValueType();
529
530 // Handle the vector case separately.
531 if (ValueVT.isVector())
532 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
533 CallConv);
534
535 unsigned OrigNumParts = NumParts;
537 "Copying to an illegal type!");
538
539 if (NumParts == 0)
540 return;
541
542 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
543 EVT PartEVT = PartVT;
544 if (PartEVT == ValueVT) {
545 assert(NumParts == 1 && "No-op copy with multiple parts!");
546 Parts[0] = Val;
547 return;
548 }
549
550 unsigned PartBits = PartVT.getSizeInBits();
551 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
552 // If the parts cover more bits than the value has, promote the value.
553 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
554 assert(NumParts == 1 && "Do not know what to promote to!");
555 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
556 } else {
557 if (ValueVT.isFloatingPoint()) {
558 // FP values need to be bitcast, then extended if they are being put
559 // into a larger container.
560 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
561 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
562 }
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
567 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
570 }
571 } else if (PartBits == ValueVT.getSizeInBits()) {
572 // Different types of the same size.
573 assert(NumParts == 1 && PartEVT != ValueVT);
574 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
575 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
576 // If the parts cover less bits than value has, truncate the value.
577 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
578 ValueVT.isInteger() &&
579 "Unknown mismatch!");
580 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
581 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
582 if (PartVT == MVT::x86mmx)
583 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
584 }
585
586 // The value may have changed - recompute ValueVT.
587 ValueVT = Val.getValueType();
588 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
589 "Failed to tile the value with PartVT!");
590
591 if (NumParts == 1) {
592 if (PartEVT != ValueVT) {
594 "scalar-to-vector conversion failed");
595 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
596 }
597
598 Parts[0] = Val;
599 return;
600 }
601
602 // Expand the value into multiple parts.
603 if (NumParts & (NumParts - 1)) {
604 // The number of parts is not a power of 2. Split off and copy the tail.
605 assert(PartVT.isInteger() && ValueVT.isInteger() &&
606 "Do not know what to expand to!");
607 unsigned RoundParts = llvm::bit_floor(NumParts);
608 unsigned RoundBits = RoundParts * PartBits;
609 unsigned OddParts = NumParts - RoundParts;
610 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
611 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
612
613 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
614 CallConv);
615
616 if (DAG.getDataLayout().isBigEndian())
617 // The odd parts were reversed by getCopyToParts - unreverse them.
618 std::reverse(Parts + RoundParts, Parts + NumParts);
619
620 NumParts = RoundParts;
621 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
622 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
623 }
624
625 // The number of parts is a power of 2. Repeatedly bisect the value using
626 // EXTRACT_ELEMENT.
627 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
629 ValueVT.getSizeInBits()),
630 Val);
631
632 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
633 for (unsigned i = 0; i < NumParts; i += StepSize) {
634 unsigned ThisBits = StepSize * PartBits / 2;
635 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
636 SDValue &Part0 = Parts[i];
637 SDValue &Part1 = Parts[i+StepSize/2];
638
639 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
640 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
641 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
642 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
643
644 if (ThisBits == PartBits && ThisVT != PartVT) {
645 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
646 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
647 }
648 }
649 }
650
651 if (DAG.getDataLayout().isBigEndian())
652 std::reverse(Parts, Parts + OrigNumParts);
653}
654
656 const SDLoc &DL, EVT PartVT) {
657 if (!PartVT.isVector())
658 return SDValue();
659
660 EVT ValueVT = Val.getValueType();
661 EVT PartEVT = PartVT.getVectorElementType();
662 EVT ValueEVT = ValueVT.getVectorElementType();
663 ElementCount PartNumElts = PartVT.getVectorElementCount();
664 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
665
666 // We only support widening vectors with equivalent element types and
667 // fixed/scalable properties. If a target needs to widen a fixed-length type
668 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
669 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
670 PartNumElts.isScalable() != ValueNumElts.isScalable())
671 return SDValue();
672
673 // Have a try for bf16 because some targets share its ABI with fp16.
674 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
676 "Cannot widen to illegal type");
677 Val = DAG.getNode(
679 ValueVT.changeVectorElementType(*DAG.getContext(), MVT::f16), Val);
680 } else if (PartEVT != ValueEVT) {
681 return SDValue();
682 }
683
684 // Widening a scalable vector to another scalable vector is done by inserting
685 // the vector into a larger undef one.
686 if (PartNumElts.isScalable())
687 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
688 Val, DAG.getVectorIdxConstant(0, DL));
689
690 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
691 // undef elements.
693 DAG.ExtractVectorElements(Val, Ops);
694 SDValue EltUndef = DAG.getUNDEF(PartEVT);
695 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
696
697 // FIXME: Use CONCAT for 2x -> 4x.
698 return DAG.getBuildVector(PartVT, DL, Ops);
699}
700
701/// getCopyToPartsVector - Create a series of nodes that contain the specified
702/// value split into legal parts.
703static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
704 SDValue Val, SDValue *Parts, unsigned NumParts,
705 MVT PartVT, const Value *V,
706 std::optional<CallingConv::ID> CallConv) {
707 EVT ValueVT = Val.getValueType();
708 assert(ValueVT.isVector() && "Not a vector");
710 const bool IsABIRegCopy = CallConv.has_value();
711
712 if (NumParts == 1) {
713 EVT PartEVT = PartVT;
714 if (PartEVT == ValueVT) {
715 // Nothing to do.
716 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
717 // Bitconvert vector->vector case.
718 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
719 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
720 Val = Widened;
721 } else if (PartVT.isVector() &&
723 ValueVT.getVectorElementType()) &&
724 PartEVT.getVectorElementCount() ==
725 ValueVT.getVectorElementCount()) {
726
727 // Promoted vector extract
728 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
729 } else if (PartEVT.isVector() &&
730 PartEVT.getVectorElementType() !=
731 ValueVT.getVectorElementType() &&
732 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
734 // Combination of widening and promotion.
735 EVT WidenVT =
737 PartVT.getVectorElementCount());
738 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
739 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
740 } else {
741 // Don't extract an integer from a float vector. This can happen if the
742 // FP type gets softened to integer and then promoted. The promotion
743 // prevents it from being picked up by the earlier bitcast case.
744 if (ValueVT.getVectorElementCount().isScalar() &&
745 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
746 // If we reach this condition and PartVT is FP, this means that
747 // ValueVT is also FP and both have a different size, otherwise we
748 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
749 // would be invalid since that would mean the smaller FP type has to
750 // be extended to the larger one.
751 if (PartVT.isFloatingPoint()) {
752 Val = DAG.getBitcast(ValueVT.getScalarType(), Val);
753 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
754 } else
755 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
756 DAG.getVectorIdxConstant(0, DL));
757 } else {
758 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
759 assert(PartVT.getFixedSizeInBits() > ValueSize &&
760 "lossy conversion of vector to scalar type");
761 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
762 Val = DAG.getBitcast(IntermediateType, Val);
763 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
764 }
765 }
766
767 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
768 Parts[0] = Val;
769 return;
770 }
771
772 // Handle a multi-element vector.
773 EVT IntermediateVT;
774 MVT RegisterVT;
775 unsigned NumIntermediates;
776 unsigned NumRegs;
777 if (IsABIRegCopy) {
778 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
779 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
780 RegisterVT);
781 } else {
782 NumRegs =
783 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
784 NumIntermediates, RegisterVT);
785 }
786
787 ensureMatchedVecRegParts(NumRegs, NumParts, RegisterVT, PartVT);
788 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
789 "Mixing scalable and fixed vectors when copying in parts");
790
791 std::optional<ElementCount> DestEltCnt;
792
793 if (IntermediateVT.isVector())
794 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
795 else
796 DestEltCnt = ElementCount::getFixed(NumIntermediates);
797
798 EVT BuiltVectorTy = EVT::getVectorVT(
799 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
800
801 if (ValueVT == BuiltVectorTy) {
802 // Nothing to do.
803 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
804 // Bitconvert vector->vector case.
805 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
806 } else {
807 if (BuiltVectorTy.getVectorElementType().bitsGT(
808 ValueVT.getVectorElementType())) {
809 // Integer promotion.
810 ValueVT = EVT::getVectorVT(*DAG.getContext(),
811 BuiltVectorTy.getVectorElementType(),
812 ValueVT.getVectorElementCount());
813 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
814 }
815
816 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
817 Val = Widened;
818 }
819 }
820
821 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
822
823 // Split the vector into intermediate operands.
824 SmallVector<SDValue, 8> Ops(NumIntermediates);
825 for (unsigned i = 0; i != NumIntermediates; ++i) {
826 if (IntermediateVT.isVector()) {
827 // This does something sensible for scalable vectors - see the
828 // definition of EXTRACT_SUBVECTOR for further details.
829 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
830 Ops[i] =
831 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
832 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
833 } else {
834 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
835 DAG.getVectorIdxConstant(i, DL));
836 }
837 }
838
839 // Split the intermediate operands into legal parts.
840 if (NumParts == NumIntermediates) {
841 // If the register was not expanded, promote or copy the value,
842 // as appropriate.
843 for (unsigned i = 0; i != NumParts; ++i)
844 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
845 } else if (NumParts > 0) {
846 // If the intermediate type was expanded, split each the value into
847 // legal parts.
848 assert(NumIntermediates != 0 && "division by zero");
849 assert(NumParts % NumIntermediates == 0 &&
850 "Must expand into a divisible number of parts!");
851 unsigned Factor = NumParts / NumIntermediates;
852 for (unsigned i = 0; i != NumIntermediates; ++i)
853 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
854 CallConv);
855 }
856}
857
858static void failForInvalidBundles(const CallBase &I, StringRef Name,
859 ArrayRef<uint32_t> AllowedBundles) {
860 if (I.hasOperandBundlesOtherThan(AllowedBundles)) {
861 ListSeparator LS;
862 std::string Error;
864 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
865 OperandBundleUse U = I.getOperandBundleAt(i);
866 if (!is_contained(AllowedBundles, U.getTagID()))
867 OS << LS << U.getTagName();
868 }
870 Twine("cannot lower ", Name)
871 .concat(Twine(" with arbitrary operand bundles: ", Error)));
872 }
873}
874
876 EVT valuevt, std::optional<CallingConv::ID> CC)
877 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
878 RegCount(1, regs.size()), CallConv(CC) {}
879
881 const DataLayout &DL, Register Reg, Type *Ty,
882 std::optional<CallingConv::ID> CC) {
884
885 CallConv = CC;
886
887 for (EVT ValueVT : ValueVTs) {
888 unsigned NumRegs =
890 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
891 : TLI.getNumRegisters(Context, ValueVT);
892 MVT RegisterVT =
894 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
895 : TLI.getRegisterType(Context, ValueVT);
896 for (unsigned i = 0; i != NumRegs; ++i)
897 Regs.push_back(Reg + i);
898 RegVTs.push_back(RegisterVT);
899 RegCount.push_back(NumRegs);
900 Reg = Reg.id() + NumRegs;
901 }
902}
903
905 FunctionLoweringInfo &FuncInfo,
906 const SDLoc &dl, SDValue &Chain,
907 SDValue *Glue, const Value *V) const {
908 // A Value with type {} or [0 x %t] needs no registers.
909 if (ValueVTs.empty())
910 return SDValue();
911
913
914 // Assemble the legal parts into the final values.
917 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
918 // Copy the legal parts from the registers.
919 EVT ValueVT = ValueVTs[Value];
920 unsigned NumRegs = RegCount[Value];
921 MVT RegisterVT = isABIMangled()
922 ? TLI.getRegisterTypeForCallingConv(
923 *DAG.getContext(), *CallConv, RegVTs[Value])
924 : RegVTs[Value];
925
926 Parts.resize(NumRegs);
927 for (unsigned i = 0; i != NumRegs; ++i) {
928 SDValue P;
929 if (!Glue) {
930 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
931 } else {
932 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
933 *Glue = P.getValue(2);
934 }
935
936 Chain = P.getValue(1);
937 Parts[i] = P;
938
939 // If the source register was virtual and if we know something about it,
940 // add an assert node.
941 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
942 continue;
943
945 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
946 if (!LOI)
947 continue;
948
949 unsigned RegSize = RegisterVT.getScalarSizeInBits();
950 unsigned NumSignBits = LOI->NumSignBits;
951 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
952
953 if (NumZeroBits == RegSize) {
954 // The current value is a zero.
955 // Explicitly express that as it would be easier for
956 // optimizations to kick in.
957 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
958 continue;
959 }
960
961 // FIXME: We capture more information than the dag can represent. For
962 // now, just use the tightest assertzext/assertsext possible.
963 bool isSExt;
964 EVT FromVT(MVT::Other);
965 if (NumZeroBits) {
966 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
967 isSExt = false;
968 } else if (NumSignBits > 1) {
969 FromVT =
970 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
971 isSExt = true;
972 } else {
973 continue;
974 }
975 // Add an assertion node.
976 assert(FromVT != MVT::Other);
977 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
978 RegisterVT, P, DAG.getValueType(FromVT));
979 }
980
981 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
982 RegisterVT, ValueVT, V, Chain, CallConv);
983 Part += NumRegs;
984 Parts.clear();
985 }
986
987 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
988}
989
991 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
992 const Value *V,
993 ISD::NodeType PreferredExtendType) const {
995 ISD::NodeType ExtendKind = PreferredExtendType;
996
997 // Get the list of the values's legal parts.
998 unsigned NumRegs = Regs.size();
999 SmallVector<SDValue, 8> Parts(NumRegs);
1000 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
1001 unsigned NumParts = RegCount[Value];
1002
1003 MVT RegisterVT = isABIMangled()
1004 ? TLI.getRegisterTypeForCallingConv(
1005 *DAG.getContext(), *CallConv, RegVTs[Value])
1006 : RegVTs[Value];
1007
1009 if (TLI.isZExtFree(peekThroughFreeze(Val), RegisterVT))
1011
1012 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
1013 NumParts, RegisterVT, V, CallConv, ExtendKind);
1014 Part += NumParts;
1015 }
1016
1017 // Copy the parts into the registers.
1018 SmallVector<SDValue, 8> Chains(NumRegs);
1019 for (unsigned i = 0; i != NumRegs; ++i) {
1020 SDValue Part;
1021 if (!Glue) {
1022 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
1023 } else {
1024 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
1025 *Glue = Part.getValue(1);
1026 }
1027
1028 Chains[i] = Part.getValue(0);
1029 }
1030
1031 if (NumRegs == 1 || Glue)
1032 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1033 // flagged to it. That is the CopyToReg nodes and the user are considered
1034 // a single scheduling unit. If we create a TokenFactor and return it as
1035 // chain, then the TokenFactor is both a predecessor (operand) of the
1036 // user as well as a successor (the TF operands are flagged to the user).
1037 // c1, f1 = CopyToReg
1038 // c2, f2 = CopyToReg
1039 // c3 = TokenFactor c1, c2
1040 // ...
1041 // = op c3, ..., f2
1042 Chain = Chains[NumRegs-1];
1043 else
1044 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1045}
1046
1048 unsigned MatchingIdx, const SDLoc &dl,
1049 SelectionDAG &DAG,
1050 std::vector<SDValue> &Ops) const {
1052
1053 InlineAsm::Flag Flag(Code, Regs.size());
1054 if (HasMatching)
1055 Flag.setMatchingOp(MatchingIdx);
1056 else if (!Regs.empty() && Regs.front().isVirtual()) {
1057 // Put the register class of the virtual registers in the flag word. That
1058 // way, later passes can recompute register class constraints for inline
1059 // assembly as well as normal instructions.
1060 // Don't do this for tied operands that can use the regclass information
1061 // from the def.
1063 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1064 Flag.setRegClass(RC->getID());
1065 }
1066
1067 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1068 Ops.push_back(Res);
1069
1070 if (Code == InlineAsm::Kind::Clobber) {
1071 // Clobbers should always have a 1:1 mapping with registers, and may
1072 // reference registers that have illegal (e.g. vector) types. Hence, we
1073 // shouldn't try to apply any sort of splitting logic to them.
1074 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1075 "No 1:1 mapping from clobbers to regs?");
1076 Register SP = TLI.getStackPointerRegisterToSaveRestore();
1077 (void)SP;
1078 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1079 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1080 assert(
1081 (Regs[I] != SP ||
1083 "If we clobbered the stack pointer, MFI should know about it.");
1084 }
1085 return;
1086 }
1087
1088 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1089 MVT RegisterVT = RegVTs[Value];
1090 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1091 RegisterVT);
1092 for (unsigned i = 0; i != NumRegs; ++i) {
1093 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1094 Register TheReg = Regs[Reg++];
1095 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1096 }
1097 }
1098}
1099
1103 unsigned I = 0;
1104 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1105 unsigned RegCount = std::get<0>(CountAndVT);
1106 MVT RegisterVT = std::get<1>(CountAndVT);
1107 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1108 for (unsigned E = I + RegCount; I != E; ++I)
1109 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1110 }
1111 return OutVec;
1112}
1113
1115 AssumptionCache *ac, const TargetLibraryInfo *li,
1116 const TargetTransformInfo &TTI) {
1117 BatchAA = aa;
1118 AC = ac;
1119 GFI = gfi;
1120 LibInfo = li;
1121 Context = DAG.getContext();
1122 LPadToCallSiteMap.clear();
1123 this->TTI = &TTI;
1124 SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1125 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1126 *DAG.getMachineFunction().getFunction().getParent());
1127 CanDescribeGlobalAddressInLocationList =
1128 canDescribeGlobalAddressInLocationList(DAG.getMachineFunction());
1129}
1130
1132 NodeMap.clear();
1133 UnusedArgNodeMap.clear();
1134 PendingLoads.clear();
1135 PendingExports.clear();
1136 PendingConstrainedFP.clear();
1137 PendingConstrainedFPStrict.clear();
1138 CurInst = nullptr;
1139 HasTailCall = false;
1140 SDNodeOrder = LowestSDNodeOrder;
1141 StatepointLowering.clear();
1142}
1143
1145 DanglingDebugInfoMap.clear();
1146}
1147
1148// Update DAG root to include dependencies on Pending chains.
1149SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1150 SDValue Root = DAG.getRoot();
1151
1152 if (Pending.empty())
1153 return Root;
1154
1155 // Add current root to PendingChains, unless we already indirectly
1156 // depend on it.
1157 if (Root.getOpcode() != ISD::EntryToken) {
1158 unsigned i = 0, e = Pending.size();
1159 for (; i != e; ++i) {
1160 assert(Pending[i].getNode()->getNumOperands() > 1);
1161 if (Pending[i].getNode()->getOperand(0) == Root)
1162 break; // Don't add the root if we already indirectly depend on it.
1163 }
1164
1165 if (i == e)
1166 Pending.push_back(Root);
1167 }
1168
1169 if (Pending.size() == 1)
1170 Root = Pending[0];
1171 else
1172 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1173
1174 DAG.setRoot(Root);
1175 Pending.clear();
1176 return Root;
1177}
1178
1182
1184 // If the new exception behavior differs from that of the pending
1185 // ones, chain up them and update the root.
1186 switch (EB) {
1189 // Floating-point exceptions produced by such operations are not intended
1190 // to be observed, so the sequence of these operations does not need to be
1191 // preserved.
1192 //
1193 // They however must not be mixed with the instructions that have strict
1194 // exception behavior. Placing an operation with 'ebIgnore' behavior between
1195 // 'ebStrict' operations could distort the observed exception behavior.
1196 if (!PendingConstrainedFPStrict.empty()) {
1197 assert(PendingConstrainedFP.empty());
1198 updateRoot(PendingConstrainedFPStrict);
1199 }
1200 break;
1202 // Floating-point exception produced by these operations may be observed, so
1203 // they must be correctly chained. If trapping on FP exceptions is
1204 // disabled, the exceptions can be observed only by functions that read
1205 // exception flags, like 'llvm.get_fpenv' or 'fetestexcept'. It means that
1206 // the order of operations is not significant between barriers.
1207 //
1208 // If trapping is enabled, each operation becomes an implicit observation
1209 // point, so the operations must be sequenced according their original
1210 // source order.
1211 if (!PendingConstrainedFP.empty()) {
1212 assert(PendingConstrainedFPStrict.empty());
1213 updateRoot(PendingConstrainedFP);
1214 }
1215 // TODO: Add support for trapping-enabled scenarios.
1216 }
1217 return DAG.getRoot();
1218}
1219
1221 // Chain up all pending constrained intrinsics together with all
1222 // pending loads, by simply appending them to PendingLoads and
1223 // then calling getMemoryRoot().
1224 PendingLoads.reserve(PendingLoads.size() +
1225 PendingConstrainedFP.size() +
1226 PendingConstrainedFPStrict.size());
1227 PendingLoads.append(PendingConstrainedFP.begin(),
1228 PendingConstrainedFP.end());
1229 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1230 PendingConstrainedFPStrict.end());
1231 PendingConstrainedFP.clear();
1232 PendingConstrainedFPStrict.clear();
1233 return getMemoryRoot();
1234}
1235
1237 // We need to emit pending fpexcept.strict constrained intrinsics,
1238 // so append them to the PendingExports list.
1239 PendingExports.append(PendingConstrainedFPStrict.begin(),
1240 PendingConstrainedFPStrict.end());
1241 PendingConstrainedFPStrict.clear();
1242 return updateRoot(PendingExports);
1243}
1244
1246 DILocalVariable *Variable,
1248 DebugLoc DL) {
1249 assert(Variable && "Missing variable");
1250
1251 // Check if address has undef value.
1252 if (!Address || isa<UndefValue>(Address) ||
1253 (Address->use_empty() && !isa<Argument>(Address))) {
1254 LLVM_DEBUG(
1255 dbgs()
1256 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1257 return;
1258 }
1259
1260 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1261
1262 SDValue &N = NodeMap[Address];
1263 if (!N.getNode() && isa<Argument>(Address))
1264 // Check unused arguments map.
1265 N = UnusedArgNodeMap[Address];
1266 SDDbgValue *SDV;
1267 if (N.getNode()) {
1268 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1269 Address = BCI->getOperand(0);
1270 // Parameters are handled specially.
1271 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1272 if (IsParameter && FINode) {
1273 // Byval parameter. We have a frame index at this point.
1274 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1275 /*IsIndirect*/ true, DL, SDNodeOrder);
1276 } else if (isa<Argument>(Address)) {
1277 // Address is an argument, so try to emit its dbg value using
1278 // virtual register info from the FuncInfo.ValueMap.
1279 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1280 FuncArgumentDbgValueKind::Declare, N);
1281 return;
1282 } else {
1283 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1284 true, DL, SDNodeOrder);
1285 }
1286 DAG.AddDbgValue(SDV, IsParameter);
1287 } else {
1288 // If Address is an argument then try to emit its dbg value using
1289 // virtual register info from the FuncInfo.ValueMap.
1290 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1291 FuncArgumentDbgValueKind::Declare, N)) {
1292 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1293 << " (could not emit func-arg dbg_value)\n");
1294 }
1295 }
1296}
1297
1299 // Add SDDbgValue nodes for any var locs here. Do so before updating
1300 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1301 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1302 // Add SDDbgValue nodes for any var locs here. Do so before updating
1303 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1304 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1305 It != End; ++It) {
1306 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1307 dropDanglingDebugInfo(Var, It->Expr);
1308 if (It->Values.isKillLocation(It->Expr)) {
1309 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1310 continue;
1311 }
1312 SmallVector<Value *> Values(It->Values.location_ops());
1313 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1314 It->Values.hasArgList())) {
1315 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1317 FnVarLocs->getDILocalVariable(It->VariableID),
1318 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1319 }
1320 }
1321 }
1322
1323 // We must skip DbgVariableRecords if they've already been processed above as
1324 // we have just emitted the debug values resulting from assignment tracking
1325 // analysis, making any existing DbgVariableRecords redundant (and probably
1326 // less correct). We still need to process DbgLabelRecords. This does sink
1327 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1328 // be important as it does so deterministcally and ordering between
1329 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1330 // printing).
1331 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1332 // Is there is any debug-info attached to this instruction, in the form of
1333 // DbgRecord non-instruction debug-info records.
1334 for (DbgRecord &DR : I.getDbgRecordRange()) {
1335 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1336 assert(DLR->getLabel() && "Missing label");
1337 SDDbgLabel *SDV =
1338 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1339 DAG.AddDbgLabel(SDV);
1340 continue;
1341 }
1342
1343 if (SkipDbgVariableRecords)
1344 continue;
1346 DILocalVariable *Variable = DVR.getVariable();
1349
1351 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1352 continue;
1353 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1354 << "\n");
1356 DVR.getDebugLoc());
1357 continue;
1358 }
1359
1360 // A DbgVariableRecord with no locations is a kill location.
1362 if (Values.empty()) {
1364 SDNodeOrder);
1365 continue;
1366 }
1367
1368 // A DbgVariableRecord with an undef or absent location is also a kill
1369 // location.
1370 if (llvm::any_of(Values,
1371 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1373 SDNodeOrder);
1374 continue;
1375 }
1376
1377 bool IsVariadic = DVR.hasArgList();
1378 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1379 SDNodeOrder, IsVariadic)) {
1380 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1381 DVR.getDebugLoc(), SDNodeOrder);
1382 }
1383 }
1384}
1385
1387 visitDbgInfo(I);
1388
1389 // Set up outgoing PHI node register values before emitting the terminator.
1390 if (I.isTerminator()) {
1391 HandlePHINodesInSuccessorBlocks(I.getParent());
1392 }
1393
1394 ++SDNodeOrder;
1395 CurInst = &I;
1396
1397 // Set inserted listener only if required.
1398 bool NodeInserted = false;
1399 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1400 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1401 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);
1402 if (PCSectionsMD || MMRA) {
1403 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1404 DAG, [&](SDNode *) { NodeInserted = true; });
1405 }
1406
1407 visit(I.getOpcode(), I);
1408
1409 if (!I.isTerminator() && !HasTailCall &&
1410 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1412
1413 // Handle metadata.
1414 if (PCSectionsMD || MMRA) {
1415 auto It = NodeMap.find(&I);
1416 if (It != NodeMap.end()) {
1417 if (PCSectionsMD)
1418 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1419 if (MMRA)
1420 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1421 } else if (NodeInserted) {
1422 // This should not happen; if it does, don't let it go unnoticed so we can
1423 // fix it. Relevant visit*() function is probably missing a setValue().
1424 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1425 << I.getModule()->getName() << "]\n";
1426 LLVM_DEBUG(I.dump());
1427 assert(false);
1428 }
1429 }
1430
1431 CurInst = nullptr;
1432}
1433
1434void SelectionDAGBuilder::visitPHI(const PHINode &) {
1435 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1436}
1437
1438void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1439 // Note: this doesn't use InstVisitor, because it has to work with
1440 // ConstantExpr's in addition to instructions.
1441 switch (Opcode) {
1442 default: llvm_unreachable("Unknown instruction type encountered!");
1443 // Build the switch statement using the Instruction.def file.
1444#define HANDLE_INST(NUM, OPCODE, CLASS) \
1445 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1446#include "llvm/IR/Instruction.def"
1447 }
1448}
1449
1451 DILocalVariable *Variable,
1452 DebugLoc DL, unsigned Order,
1455 // For variadic dbg_values we will now insert poison.
1456 // FIXME: We can potentially recover these!
1458 for (const Value *V : Values) {
1459 auto *Poison = PoisonValue::get(V->getType());
1460 Locs.push_back(SDDbgOperand::fromConst(Poison));
1461 }
1462 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1463 /*IsIndirect=*/false, DL, Order,
1464 /*IsVariadic=*/true);
1465 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1466 return true;
1467}
1468
1470 DILocalVariable *Var,
1471 DIExpression *Expr,
1472 bool IsVariadic, DebugLoc DL,
1473 unsigned Order) {
1474 if (IsVariadic) {
1475 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1476 return;
1477 }
1478 // TODO: Dangling debug info will eventually either be resolved or produce
1479 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1480 // between the original dbg.value location and its resolved DBG_VALUE,
1481 // which we should ideally fill with an extra poison DBG_VALUE.
1482 assert(Values.size() == 1);
1483 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1484}
1485
1487 const DIExpression *Expr) {
1488 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1489 DIVariable *DanglingVariable = DDI.getVariable();
1490 DIExpression *DanglingExpr = DDI.getExpression();
1491 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1492 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1493 << printDDI(nullptr, DDI) << "\n");
1494 return true;
1495 }
1496 return false;
1497 };
1498
1499 for (auto &DDIMI : DanglingDebugInfoMap) {
1500 DanglingDebugInfoVector &DDIV = DDIMI.second;
1501
1502 // If debug info is to be dropped, run it through final checks to see
1503 // whether it can be salvaged.
1504 for (auto &DDI : DDIV)
1505 if (isMatchingDbgValue(DDI))
1506 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1507
1508 erase_if(DDIV, isMatchingDbgValue);
1509 }
1510}
1511
1512// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1513// generate the debug data structures now that we've seen its definition.
1515 SDValue Val) {
1516 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1517 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1518 return;
1519
1520 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1521 for (auto &DDI : DDIV) {
1522 DebugLoc DL = DDI.getDebugLoc();
1523 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1524 DILocalVariable *Variable = DDI.getVariable();
1525 DIExpression *Expr = DDI.getExpression();
1526 assert(Variable->isValidLocationForIntrinsic(DL) &&
1527 "Expected inlined-at fields to agree");
1528 SDDbgValue *SDV;
1529 if (Val.getNode()) {
1530 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1531 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1532 // we couldn't resolve it directly when examining the DbgValue intrinsic
1533 // in the first place we should not be more successful here). Unless we
1534 // have some test case that prove this to be correct we should avoid
1535 // calling EmitFuncArgumentDbgValue here.
1536 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1537 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1538 FuncArgumentDbgValueKind::Value, Val)) {
1539 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1540 << printDDI(V, DDI) << "\n");
1541 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1542 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1543 // inserted after the definition of Val when emitting the instructions
1544 // after ISel. An alternative could be to teach
1545 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1546 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1547 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1548 << ValSDNodeOrder << "\n");
1549 SDV = getDbgValue(Val, Variable, Expr, DL,
1550 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1551 DAG.AddDbgValue(SDV, false);
1552 } else
1553 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1554 << printDDI(V, DDI)
1555 << " in EmitFuncArgumentDbgValue\n");
1556 } else {
1557 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1558 << "\n");
1559 auto Poison = PoisonValue::get(V->getType());
1560 auto SDV =
1561 DAG.getConstantDbgValue(Variable, Expr, Poison, DL, DbgSDNodeOrder);
1562 DAG.AddDbgValue(SDV, false);
1563 }
1564 }
1565 DDIV.clear();
1566}
1567
1568/// If \p V is the address of a describable global, possibly displaced by a
1569/// constant, return the global and fold the displacement into location operand
1570/// \p OpIdx of \p Expr. The displacement rides along in the expression rather
1571/// than in the operand, so that it survives into a DBG_INSTR_REF.
1572static const GlobalValue *
1573getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx,
1574 const MachineFunction &MF) {
1575 const auto *C = dyn_cast<Constant>(V);
1576 if (!C)
1577 return nullptr;
1578 int64_t Offset;
1580 if (GV && Offset) {
1583 Expr = DIExpression::appendOpsToArg(Expr, Ops, OpIdx, /*StackValue=*/false);
1584 }
1585 return GV;
1586}
1587
1589 DanglingDebugInfo &DDI) {
1590 // TODO: For the variadic implementation, instead of only checking the fail
1591 // state of `handleDebugValue`, we need know specifically which values were
1592 // invalid, so that we attempt to salvage only those values when processing
1593 // a DIArgList.
1594 const Value *OrigV = V;
1595 DILocalVariable *Var = DDI.getVariable();
1596 DIExpression *Expr = DDI.getExpression();
1597 DebugLoc DL = DDI.getDebugLoc();
1598 unsigned SDOrder = DDI.getSDNodeOrder();
1599
1600 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1601 // that DW_OP_stack_value is desired.
1602 bool StackValue = true;
1603
1604 // handleDebugValue holds out for a register with the address of a global
1605 // that a location list could not name. With no such register forthcoming,
1606 // naming the global still beats dropping the location.
1607 auto HandleGlobalAddress = [&] {
1608 DIExpression *GVExpr = Expr;
1609 const GlobalValue *GV =
1610 getGlobalAddressDbgOperand(V, GVExpr, 0, DAG.getMachineFunction());
1611 if (!GV)
1612 return false;
1613 SDDbgValue *SDV = DAG.getDbgValueList(
1614 Var, GVExpr, SDDbgOperand::fromGlobalAddr(GV), /*Dependencies=*/{},
1615 /*IsIndirect=*/false, DL, SDOrder, /*IsVariadic=*/false);
1616 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1617 return true;
1618 };
1619
1620 // Can this Value can be encoded without any further work?
1621 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false) ||
1622 HandleGlobalAddress())
1623 return;
1624
1625 // Attempt to salvage back through as many instructions as possible. Bail if
1626 // a non-instruction is seen, such as a constant expression or global
1627 // variable. FIXME: Further work could recover those too.
1628 while (isa<Instruction>(V)) {
1629 const Instruction &VAsInst = *cast<const Instruction>(V);
1630 // Temporary "0", awaiting real implementation.
1632 SmallVector<Value *, 4> AdditionalValues;
1633 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1634 Expr->getNumLocationOperands(), Ops,
1635 AdditionalValues);
1636 // If we cannot salvage any further, and haven't yet found a suitable debug
1637 // expression, bail out.
1638 if (!V)
1639 break;
1640
1641 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1642 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1643 // here for variadic dbg_values, remove that condition.
1644 if (!AdditionalValues.empty())
1645 break;
1646
1647 // New value and expr now represent this debuginfo.
1649
1650 // Some kind of simplification occurred: check whether the operand of the
1651 // salvaged debug expression can be encoded in this DAG.
1652 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false) ||
1653 HandleGlobalAddress()) {
1654 LLVM_DEBUG(
1655 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1656 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1657 return;
1658 }
1659 }
1660
1661 // This was the final opportunity to salvage this debug information, and it
1662 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1663 // any earlier variable location.
1664 assert(OrigV && "V shouldn't be null");
1665 auto *Poison = PoisonValue::get(OrigV->getType());
1666 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Poison, DL, SDOrder);
1667 DAG.AddDbgValue(SDV, false);
1668 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1669 << printDDI(OrigV, DDI) << "\n");
1670}
1671
1673 DIExpression *Expr,
1674 DebugLoc DbgLoc,
1675 unsigned Order) {
1676 Value *Poison = PoisonValue::get(Type::getInt1Ty(*Context));
1679 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1680 /*IsVariadic*/ false);
1681}
1682
1684 DILocalVariable *Var,
1685 DIExpression *Expr, DebugLoc DbgLoc,
1686 unsigned Order, bool IsVariadic) {
1687 if (Values.empty())
1688 return true;
1689
1690 // Filter EntryValue locations out early.
1691 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1692 return true;
1693
1694 SmallVector<SDDbgOperand> LocationOps;
1695 SmallVector<SDNode *> Dependencies;
1696 for (const auto &[OpIdx, V] : enumerate(Values)) {
1697 // Constant value.
1700 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1701 continue;
1702 }
1703
1704 // Look through IntToPtr constants.
1705 if (auto *CE = dyn_cast<ConstantExpr>(V))
1706 if (CE->getOpcode() == Instruction::IntToPtr) {
1707 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1708 continue;
1709 }
1710
1711 // The address of a global is a link-time constant, and so is a constant
1712 // displacement from one. A global whose address cannot be described this
1713 // way falls through to be described by whatever materializes it instead.
1714 // So does one that a location list could not name, should the variable
1715 // need one; salvageUnresolvedDbgValue names it if nothing materializes it.
1716 if (CanDescribeGlobalAddressInLocationList)
1718 V, Expr, OpIdx, DAG.getMachineFunction())) {
1720 continue;
1721 }
1722
1723 // If the Value is a frame index, we can create a FrameIndex debug value
1724 // without relying on the DAG at all.
1725 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1726 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1727 if (SI != FuncInfo.StaticAllocaMap.end()) {
1728 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1729 continue;
1730 }
1731 }
1732
1733 // Do not use getValue() in here; we don't want to generate code at
1734 // this point if it hasn't been done yet.
1735 SDValue N = NodeMap[V];
1736 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1737 N = UnusedArgNodeMap[V];
1738
1739 if (N.getNode()) {
1740 // Only emit func arg dbg value for non-variadic dbg.values for now.
1741 if (!IsVariadic &&
1742 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1743 FuncArgumentDbgValueKind::Value, N))
1744 return true;
1745 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1746 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1747 // describe stack slot locations.
1748 //
1749 // Consider "int x = 0; int *px = &x;". There are two kinds of
1750 // interesting debug values here after optimization:
1751 //
1752 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1753 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1754 //
1755 // Both describe the direct values of their associated variables.
1756 Dependencies.push_back(N.getNode());
1757 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1758 continue;
1759 }
1760 LocationOps.emplace_back(
1761 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1762 continue;
1763 }
1764
1765 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1766 // Special rules apply for the first dbg.values of parameter variables in a
1767 // function. Identify them by the fact they reference Argument Values, that
1768 // they're parameters, and they are parameters of the current function. We
1769 // need to let them dangle until they get an SDNode.
1770 bool IsParamOfFunc =
1771 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1772 if (IsParamOfFunc)
1773 return false;
1774
1775 // The value is not used in this block yet (or it would have an SDNode).
1776 // We still want the value to appear for the user if possible -- if it has
1777 // an associated VReg, we can refer to that instead.
1778 auto VMI = FuncInfo.ValueMap.find(V);
1779 if (VMI != FuncInfo.ValueMap.end()) {
1780 Register Reg = VMI->second;
1781 // If this is a PHI node, it may be split up into several MI PHI nodes
1782 // (in FunctionLoweringInfo::set).
1783 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1784 V->getType(), std::nullopt);
1785 if (RFV.occupiesMultipleRegs()) {
1786 // FIXME: We could potentially support variadic dbg_values here.
1787 if (IsVariadic)
1788 return false;
1789 unsigned Offset = 0;
1790 unsigned BitsToDescribe = 0;
1791 if (auto VarSize = Var->getSizeInBits())
1792 BitsToDescribe = *VarSize;
1793 if (auto Fragment = Expr->getFragmentInfo())
1794 BitsToDescribe = Fragment->SizeInBits;
1795 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1796 // Bail out if all bits are described already.
1797 if (Offset >= BitsToDescribe)
1798 break;
1799 // TODO: handle scalable vectors.
1800 unsigned RegisterSize = RegAndSize.second;
1801 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1802 ? BitsToDescribe - Offset
1803 : RegisterSize;
1804 auto FragmentExpr = DIExpression::createFragmentExpression(
1805 Expr, Offset, FragmentSize);
1806 if (!FragmentExpr)
1807 continue;
1808 SDDbgValue *SDV = DAG.getVRegDbgValue(
1809 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);
1810 DAG.AddDbgValue(SDV, false);
1811 Offset += RegisterSize;
1812 }
1813 return true;
1814 }
1815 // We can use simple vreg locations for variadic dbg_values as well.
1816 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1817 continue;
1818 }
1819 // We failed to create a SDDbgOperand for V.
1820 return false;
1821 }
1822
1823 // We have created a SDDbgOperand for each Value in Values.
1824 assert(!LocationOps.empty());
1825 SDDbgValue *SDV =
1826 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1827 /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
1828 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1829 return true;
1830}
1831
1833 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1834 for (auto &Pair : DanglingDebugInfoMap)
1835 for (auto &DDI : Pair.second)
1836 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1838}
1839
1840/// getCopyFromRegs - If there was virtual register allocated for the value V
1841/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1843 auto It = FuncInfo.ValueMap.find(V);
1844 SDValue Result;
1845
1846 if (It != FuncInfo.ValueMap.end()) {
1847 Register InReg = It->second;
1848
1849 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1850 DAG.getDataLayout(), InReg, Ty,
1851 std::nullopt); // This is not an ABI copy.
1852 SDValue Chain = DAG.getEntryNode();
1853 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1854 V);
1855 resolveDanglingDebugInfo(V, Result);
1856 }
1857
1858 return Result;
1859}
1860
1861/// getValue - Return an SDValue for the given Value.
1863 // If we already have an SDValue for this value, use it. It's important
1864 // to do this first, so that we don't create a CopyFromReg if we already
1865 // have a regular SDValue.
1866 SDValue &N = NodeMap[V];
1867 if (N.getNode()) return N;
1868
1869 // If there's a virtual register allocated and initialized for this
1870 // value, use it.
1871 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1872 return copyFromReg;
1873
1874 // Otherwise create a new SDValue and remember it.
1875 SDValue Val = getValueImpl(V);
1876 NodeMap[V] = Val;
1878 return Val;
1879}
1880
1881void SelectionDAGBuilder::setValueToPoison(const Value *V, const SDLoc &dl) {
1882 if (V->getType()->isVoidTy())
1883 return;
1884
1885 SmallVector<EVT, 4> ValueVTs;
1886 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
1887 V->getType(), ValueVTs);
1888 if (ValueVTs.empty())
1889 return;
1890 setValue(V, DAG.getErrorMergeValues(ValueVTs, SDValue(), dl));
1891}
1892
1893/// getNonRegisterValue - Return an SDValue for the given Value, but
1894/// don't look in FuncInfo.ValueMap for a virtual register.
1896 // If we already have an SDValue for this value, use it.
1897 SDValue &N = NodeMap[V];
1898 if (N.getNode()) {
1899 if (isIntOrFPConstant(N)) {
1900 // Remove the debug location from the node as the node is about to be used
1901 // in a location which may differ from the original debug location. This
1902 // is relevant to Constant and ConstantFP nodes because they can appear
1903 // as constant expressions inside PHI nodes.
1904 N->setDebugLoc(DebugLoc());
1905 }
1906 return N;
1907 }
1908
1909 // Otherwise create a new SDValue and remember it.
1910 SDValue Val = getValueImpl(V);
1911 NodeMap[V] = Val;
1913 return Val;
1914}
1915
1916/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1917/// Create an SDValue for the given value.
1919 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1920
1921 if (const Constant *C = dyn_cast<Constant>(V)) {
1922 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1923
1924 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) {
1925 SDLoc DL = getCurSDLoc();
1926
1927 // DAG.getConstant() may attempt to legalise the vector constant which can
1928 // significantly change the combines applied to the DAG. To reduce the
1929 // divergence when enabling ConstantInt based vectors we try to construct
1930 // the DAG in the same way as shufflevector based splats. TODO: The
1931 // divergence sometimes leads to better optimisations. Ideally we should
1932 // prevent DAG.getConstant() from legalising too early but there are some
1933 // degradations preventing this.
1934 if (VT.isScalableVector())
1935 return DAG.getNode(
1936 ISD::SPLAT_VECTOR, DL, VT,
1937 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));
1938 if (VT.isFixedLengthVector())
1939 return DAG.getSplatBuildVector(
1940 VT, DL,
1941 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));
1942 return DAG.getConstant(*CI, DL, VT);
1943 }
1944
1945 if (const ConstantByte *CB = dyn_cast<ConstantByte>(C))
1946 return DAG.getConstant(CB->getValue(), getCurSDLoc(), VT);
1947
1948 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1949 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1950
1951 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
1952 return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT,
1953 getValue(CPA->getPointer()), getValue(CPA->getKey()),
1954 getValue(CPA->getAddrDiscriminator()),
1955 getValue(CPA->getDiscriminator()));
1956 }
1957
1959 return DAG.getConstant(0, getCurSDLoc(), VT);
1960
1961 if (match(C, m_VScale()))
1962 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1963
1964 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1965 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1966
1967 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1968 return isa<PoisonValue>(C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1969
1970 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1971 visit(CE->getOpcode(), *CE);
1972 SDValue N1 = NodeMap[V];
1973 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1974 return N1;
1975 }
1976
1978 SmallVector<SDValue, 4> Constants;
1979 for (const Use &U : C->operands()) {
1980 SDNode *Val = getValue(U).getNode();
1981 // If the operand is an empty aggregate, there are no values.
1982 if (!Val) continue;
1983 // Add each leaf value from the operand to the Constants list
1984 // to form a flattened list of all the values.
1985 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1986 Constants.push_back(SDValue(Val, i));
1987 }
1988
1989 return DAG.getMergeValues(Constants, getCurSDLoc());
1990 }
1991
1992 if (const ConstantDataSequential *CDS =
1995 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1996 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1997 // Add each leaf value from the operand to the Constants list
1998 // to form a flattened list of all the values.
1999 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
2000 Ops.push_back(SDValue(Val, i));
2001 }
2002
2003 if (isa<ArrayType>(CDS->getType()))
2004 return DAG.getMergeValues(Ops, getCurSDLoc());
2005 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);
2006 }
2007
2008 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
2010 "Unknown struct or array constant!");
2011
2012 SmallVector<EVT, 4> ValueVTs;
2013 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
2014 unsigned NumElts = ValueVTs.size();
2015 if (NumElts == 0)
2016 return SDValue(); // empty struct
2017 SmallVector<SDValue, 4> Constants(NumElts);
2018 for (unsigned i = 0; i != NumElts; ++i) {
2019 EVT EltVT = ValueVTs[i];
2020 if (isa<UndefValue>(C))
2021 Constants[i] = DAG.getUNDEF(EltVT);
2022 else if (EltVT.isFloatingPoint())
2023 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
2024 else
2025 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
2026 }
2027
2028 return DAG.getMergeValues(Constants, getCurSDLoc());
2029 }
2030
2031 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
2032 return DAG.getBlockAddress(BA, VT);
2033
2034 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
2035 return getValue(Equiv->getGlobalValue());
2036
2037 if (const auto *NC = dyn_cast<NoCFIValue>(C))
2038 return getValue(NC->getGlobalValue());
2039
2040 if (VT == MVT::aarch64svcount) {
2041 assert(C->isNullValue() && "Can only zero this target type!");
2042 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
2043 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
2044 }
2045
2046 if (VT.isRISCVVectorTuple()) {
2047 assert(C->isNullValue() && "Can only zero this target type!");
2048 return DAG.getNode(
2050 DAG.getNode(
2052 EVT::getVectorVT(*DAG.getContext(), MVT::i8,
2053 VT.getSizeInBits().getKnownMinValue() / 8, true),
2054 DAG.getConstant(0, getCurSDLoc(), MVT::getIntegerVT(8))));
2055 }
2056
2057 if (VT == MVT::externref || VT == MVT::funcref) {
2058 assert(C->isNullValue() && "Can only zero this target type!");
2059 // The zero value of a WebAssembly reference type is the null reference,
2060 // materialized with ref.null.
2061 Intrinsic::ID IID = VT == MVT::externref ? Intrinsic::wasm_ref_null_extern
2062 : Intrinsic::wasm_ref_null_func;
2063 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VT,
2064 DAG.getTargetConstant(IID, getCurSDLoc(), MVT::i32));
2065 }
2066
2067 VectorType *VecTy = cast<VectorType>(V->getType());
2068
2069 // Now that we know the number and type of the elements, get that number of
2070 // elements into the Ops array based on what kind of constant it is.
2071 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
2073 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
2074 for (unsigned i = 0; i != NumElements; ++i)
2075 Ops.push_back(getValue(CV->getOperand(i)));
2076
2077 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);
2078 }
2079
2081 EVT EltVT =
2082 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
2083
2084 SDValue Op;
2085 if (EltVT.isFloatingPoint())
2086 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
2087 else
2088 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
2089
2090 return DAG.getSplat(VT, getCurSDLoc(), Op);
2091 }
2092
2093 llvm_unreachable("Unknown vector constant");
2094 }
2095
2096 // If this is a static alloca, generate it as the frameindex instead of
2097 // computation.
2098 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
2099 auto SI = FuncInfo.StaticAllocaMap.find(AI);
2100 if (SI != FuncInfo.StaticAllocaMap.end())
2101 return DAG.getFrameIndex(
2102 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
2103 }
2104
2105 // If this is an instruction which fast-isel has deferred, select it now.
2106 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
2107 Register InReg = FuncInfo.InitializeRegForValue(Inst);
2108 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
2109 Inst->getType(), std::nullopt);
2110 SDValue Chain = DAG.getEntryNode();
2111 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
2112 }
2113
2114 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
2115 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
2116
2117 if (const auto *BB = dyn_cast<BasicBlock>(V))
2118 return DAG.getBasicBlock(FuncInfo.getMBB(BB));
2119
2120 llvm_unreachable("Can't get register for value!");
2121}
2122
2123void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
2125 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
2126 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
2127 bool IsSEH = isAsynchronousEHPersonality(Pers);
2128 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
2129 if (IsSEH) {
2130 // For SEH, EHCont Guard needs to know that this catchpad is a target.
2131 CatchPadMBB->setIsEHContTarget(true);
2133 } else
2134 CatchPadMBB->setIsEHScopeEntry();
2135 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
2136 if (IsMSVCCXX || IsCoreCLR)
2137 CatchPadMBB->setIsEHFuncletEntry();
2138}
2139
2140void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
2141 // Update machine-CFG edge.
2142 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());
2143 FuncInfo.MBB->addSuccessor(TargetMBB);
2144
2145 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
2146 bool IsSEH = isAsynchronousEHPersonality(Pers);
2147 if (IsSEH) {
2148 // If this is not a fall-through branch or optimizations are switched off,
2149 // emit the branch.
2150 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
2151 TM.getOptLevel() == CodeGenOptLevel::None)
2152 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2153 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
2154 return;
2155 }
2156
2157 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2158 TargetMBB->setIsEHContTarget(true);
2159 DAG.getMachineFunction().setHasEHContTarget(true);
2160
2161 // Figure out the funclet membership for the catchret's successor.
2162 // This will be used by the FuncletLayout pass to determine how to order the
2163 // BB's.
2164 // A 'catchret' returns to the outer scope's color.
2165 Value *ParentPad = I.getCatchSwitchParentPad();
2166 const BasicBlock *SuccessorColor;
2167 if (isa<ConstantTokenNone>(ParentPad))
2168 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2169 else
2170 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
2171 assert(SuccessorColor && "No parent funclet for catchret!");
2172 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(SuccessorColor);
2173 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2174
2175 // Create the terminator node.
2176 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
2177 getControlRoot(), DAG.getBasicBlock(TargetMBB),
2178 DAG.getBasicBlock(SuccessorColorMBB));
2179 DAG.setRoot(Ret);
2180}
2181
2182void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2183 // Don't emit any special code for the cleanuppad instruction. It just marks
2184 // the start of an EH scope/funclet.
2185 FuncInfo.MBB->setIsEHScopeEntry();
2186 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
2187 if (Pers != EHPersonality::Wasm_CXX && Pers != EHPersonality::Wasm_D) {
2188 FuncInfo.MBB->setIsEHFuncletEntry();
2189 FuncInfo.MBB->setIsCleanupFuncletEntry();
2190 }
2191}
2192
2193/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2194/// many places it could ultimately go. In the IR, we have a single unwind
2195/// destination, but in the machine CFG, we enumerate all the possible blocks.
2196/// This function skips over imaginary basic blocks that hold catchswitch
2197/// instructions, and finds all the "real" machine
2198/// basic block destinations. As those destinations may not be successors of
2199/// EHPadBB, here we also calculate the edge probability to those destinations.
2200/// The passed-in Prob is the edge probability to EHPadBB.
2202 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2203 BranchProbability Prob,
2204 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2205 &UnwindDests) {
2206 EHPersonality Personality =
2208 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2209 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2210 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2211 bool IsWasmD = Personality == EHPersonality::Wasm_D;
2212 bool IsSEH = isAsynchronousEHPersonality(Personality);
2213
2214 while (EHPadBB) {
2216 BasicBlock *NewEHPadBB = nullptr;
2217 if (isa<LandingPadInst>(Pad)) {
2218 // Stop on landingpads. They are not funclets.
2219 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2220 break;
2221 } else if (isa<CleanupPadInst>(Pad)) {
2222 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2223 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2224 // which always catches an exception.
2225 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2226 UnwindDests.back().first->setIsEHScopeEntry();
2227 // In Wasm, EH scopes are not funclets
2228 if (!IsWasmCXX && !IsWasmD)
2229 UnwindDests.back().first->setIsEHFuncletEntry();
2230 break;
2231 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2232 // Add the catchpad handlers to the possible destinations.
2233 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2234 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2235 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2236 if (IsMSVCCXX || IsCoreCLR)
2237 UnwindDests.back().first->setIsEHFuncletEntry();
2238 if (!IsSEH)
2239 UnwindDests.back().first->setIsEHScopeEntry();
2240 }
2241 NewEHPadBB = CatchSwitch->getUnwindDest();
2242 } else {
2243 continue;
2244 }
2245
2246 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2247 if (BPI && NewEHPadBB)
2248 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2249 EHPadBB = NewEHPadBB;
2250 }
2251}
2252
2253void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2254 // Update successor info.
2256 auto UnwindDest = I.getUnwindDest();
2257 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2258 BranchProbability UnwindDestProb =
2259 (BPI && UnwindDest)
2260 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2262 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2263 for (auto &UnwindDest : UnwindDests) {
2264 UnwindDest.first->setIsEHPad();
2265 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2266 }
2267 FuncInfo.MBB->normalizeSuccProbs();
2268
2269 // Create the terminator node.
2270 MachineBasicBlock *CleanupPadMBB =
2271 FuncInfo.getMBB(I.getCleanupPad()->getParent());
2272 SDValue Ret = DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other,
2273 getControlRoot(), DAG.getBasicBlock(CleanupPadMBB));
2274 DAG.setRoot(Ret);
2275}
2276
2277void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2278 report_fatal_error("visitCatchSwitch not yet implemented!");
2279}
2280
2281void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2282 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2283 auto &DL = DAG.getDataLayout();
2284 SDValue Chain = getControlRoot();
2287
2288 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2289 // lower
2290 //
2291 // %val = call <ty> @llvm.experimental.deoptimize()
2292 // ret <ty> %val
2293 //
2294 // differently.
2295 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2297 return;
2298 }
2299
2300 if (!FuncInfo.CanLowerReturn) {
2301 Register DemoteReg = FuncInfo.DemoteRegister;
2302
2303 // Emit a store of the return value through the virtual register.
2304 // Leave Outs empty so that LowerReturn won't try to load return
2305 // registers the usual way.
2306 MVT PtrValueVT = TLI.getPointerTy(DL, DL.getAllocaAddrSpace());
2307 SDValue RetPtr =
2308 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVT);
2309 Type *RetTy = I.getOperand(0)->getType();
2310 Align BaseAlign = DL.getPrefTypeAlign(RetTy);
2311 RetPtr =
2312 TLI.annotateStackObjectPointer(RetPtr, DAG, getCurSDLoc(), BaseAlign);
2313 SDValue RetOp = getValue(I.getOperand(0));
2314
2315 SmallVector<EVT, 4> ValueVTs, MemVTs;
2316 SmallVector<uint64_t, 4> Offsets;
2317 ComputeValueVTs(TLI, DL, RetTy, ValueVTs, &MemVTs, &Offsets, 0);
2318 unsigned NumValues = ValueVTs.size();
2319
2320 SmallVector<SDValue, 4> Chains(NumValues);
2321 for (unsigned i = 0; i != NumValues; ++i) {
2322 // An aggregate return value cannot wrap around the address space, so
2323 // offsets to its parts don't wrap either.
2324 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
2325 TypeSize::getFixed(Offsets[i]));
2326
2327 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2328 if (MemVTs[i] != ValueVTs[i])
2329 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2330 Chains[i] = DAG.getStore(
2331 Chain, getCurSDLoc(), Val,
2332 // FIXME: better loc info would be nice.
2333 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
2334 commonAlignment(BaseAlign, Offsets[i]));
2335 }
2336
2337 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
2338 MVT::Other, Chains);
2339 } else if (I.getNumOperands() != 0) {
2341 ComputeValueTypes(DL, I.getOperand(0)->getType(), Types);
2342 unsigned NumValues = Types.size();
2343 if (NumValues) {
2344 SDValue RetOp = getValue(I.getOperand(0));
2345
2346 const Function *F = I.getParent()->getParent();
2347
2349 I.getOperand(0)->getType(), F->getCallingConv(),
2350 /*IsVarArg*/ false, DL);
2351
2353 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2355 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2357
2358 LLVMContext &Context = F->getContext();
2359 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2360
2361 for (unsigned j = 0; j != NumValues; ++j) {
2362 EVT VT = TLI.getValueType(DL, Types[j]);
2363
2364 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2366
2367 CallingConv::ID CC = F->getCallingConv();
2368
2370 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2373 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2374 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2375
2376 // 'inreg' on function refers to return value
2377 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2378 if (RetInReg)
2379 Flags.setInReg();
2380
2381 if (I.getOperand(0)->getType()->isPointerTy()) {
2382 Flags.setPointer();
2383 Flags.setPointerAddrSpace(
2384 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2385 }
2386
2387 if (NeedsRegBlock) {
2388 Flags.setInConsecutiveRegs();
2389 if (j == NumValues - 1)
2390 Flags.setInConsecutiveRegsLast();
2391 }
2392
2393 // Propagate extension type if any
2395 Flags.setSExt();
2396 else if (ExtendKind == ISD::ZERO_EXTEND)
2397 Flags.setZExt();
2398 else if (F->getAttributes().hasRetAttr(Attribute::NoExt))
2399 Flags.setNoExt();
2400
2401 for (unsigned i = 0; i < NumParts; ++i) {
2402 Outs.push_back(ISD::OutputArg(Flags,
2403 Parts[i].getValueType().getSimpleVT(),
2404 VT, Types[j], 0, 0));
2405 OutVals.push_back(Parts[i]);
2406 }
2407 }
2408 }
2409 }
2410
2411 // Push in swifterror virtual register as the last element of Outs. This makes
2412 // sure swifterror virtual register will be returned in the swifterror
2413 // physical register.
2414 const Function *F = I.getParent()->getParent();
2415 if (TLI.supportSwiftError() &&
2416 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2417 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2418 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2419 Flags.setSwiftError();
2420 Outs.push_back(ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
2421 /*argvt=*/EVT(TLI.getPointerTy(DL)),
2422 PointerType::getUnqual(*DAG.getContext()),
2423 /*origidx=*/1, /*partOffs=*/0));
2424 // Create SDNode for the swifterror virtual register.
2425 OutVals.push_back(
2426 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2427 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2428 EVT(TLI.getPointerTy(DL))));
2429 }
2430
2431 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2432 CallingConv::ID CallConv =
2433 DAG.getMachineFunction().getFunction().getCallingConv();
2434 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2435 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2436
2437 // Verify that the target's LowerReturn behaved as expected.
2438 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2439 "LowerReturn didn't return a valid chain!");
2440
2441 // Update the DAG with the new chain value resulting from return lowering.
2442 DAG.setRoot(Chain);
2443}
2444
2445/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2446/// created for it, emit nodes to copy the value into the virtual
2447/// registers.
2449 // Skip empty types
2450 if (V->getType()->isEmptyTy())
2451 return;
2452
2453 auto VMI = FuncInfo.ValueMap.find(V);
2454 if (VMI != FuncInfo.ValueMap.end()) {
2455 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2456 "Unused value assigned virtual registers!");
2457 CopyValueToVirtualRegister(V, VMI->second);
2458 }
2459}
2460
2461/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2462/// the current basic block, add it to ValueMap now so that we'll get a
2463/// CopyTo/FromReg.
2465 // No need to export constants.
2466 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2467
2468 // Already exported?
2469 if (FuncInfo.isExportedInst(V)) return;
2470
2471 Register Reg = FuncInfo.InitializeRegForValue(V);
2473}
2474
2476 const BasicBlock *FromBB) {
2477 // The operands of the setcc have to be in this block. We don't know
2478 // how to export them from some other block.
2479 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2480 // Can export from current BB.
2481 if (VI->getParent() == FromBB)
2482 return true;
2483
2484 // Is already exported, noop.
2485 return FuncInfo.isExportedInst(V);
2486 }
2487
2488 // If this is an argument, we can export it if the BB is the entry block or
2489 // if it is already exported.
2490 if (isa<Argument>(V)) {
2491 if (FromBB->isEntryBlock())
2492 return true;
2493
2494 // Otherwise, can only export this if it is already exported.
2495 return FuncInfo.isExportedInst(V);
2496 }
2497
2498 // Otherwise, constants can always be exported.
2499 return true;
2500}
2501
2502/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2504SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2505 const MachineBasicBlock *Dst) const {
2507 const BasicBlock *SrcBB = Src->getBasicBlock();
2508 const BasicBlock *DstBB = Dst->getBasicBlock();
2509 if (!BPI) {
2510 // If BPI is not available, set the default probability as 1 / N, where N is
2511 // the number of successors.
2512 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2513 return BranchProbability(1, SuccSize);
2514 }
2515 return BPI->getEdgeProbability(SrcBB, DstBB);
2516}
2517
2518void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2519 MachineBasicBlock *Dst,
2520 BranchProbability Prob) {
2521 if (!FuncInfo.BPI)
2522 Src->addSuccessorWithoutProb(Dst);
2523 else {
2524 if (Prob.isUnknown())
2525 Prob = getEdgeProbability(Src, Dst);
2526 Src->addSuccessor(Dst, Prob);
2527 }
2528}
2529
2530static bool InBlock(const Value *V, const BasicBlock *BB) {
2531 if (const Instruction *I = dyn_cast<Instruction>(V))
2532 return I->getParent() == BB;
2533 return true;
2534}
2535
2536/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2537/// This function emits a branch and is used at the leaves of an OR or an
2538/// AND operator tree.
2539void
2542 MachineBasicBlock *FBB,
2543 MachineBasicBlock *CurBB,
2544 MachineBasicBlock *SwitchBB,
2545 BranchProbability TProb,
2546 BranchProbability FProb,
2547 bool InvertCond) {
2548 const BasicBlock *BB = CurBB->getBasicBlock();
2549
2550 // If the leaf of the tree is a comparison, merge the condition into
2551 // the caseblock.
2552 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2553 // The operands of the cmp have to be in this block. We don't know
2554 // how to export them from some other block. If this is the first block
2555 // of the sequence, no exporting is needed.
2556 if (CurBB == SwitchBB ||
2557 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2558 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2559 ISD::CondCode Condition;
2560 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2561 ICmpInst::Predicate Pred =
2562 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2563 Condition = getICmpCondCode(Pred);
2564 } else {
2565 const FCmpInst *FC = cast<FCmpInst>(Cond);
2566 FCmpInst::Predicate Pred =
2567 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2568 Condition = getFCmpCondCode(Pred);
2569 if (FC->hasNoNaNs() ||
2570 (isKnownNeverNaN(FC->getOperand(0),
2571 SimplifyQuery(DAG.getDataLayout(), FC)) &&
2572 isKnownNeverNaN(FC->getOperand(1),
2573 SimplifyQuery(DAG.getDataLayout(), FC))))
2574 Condition = getFCmpCodeWithoutNaN(Condition);
2575 }
2576
2577 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2578 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2579 SL->SwitchCases.push_back(CB);
2580 return;
2581 }
2582 }
2583
2584 // Create a CaseBlock record representing this branch.
2585 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2586 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2587 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2588 SL->SwitchCases.push_back(CB);
2589}
2590
2591// Collect dependencies on V recursively. This is used for the cost analysis in
2592// `shouldKeepJumpConditionsTogether`.
2596 unsigned Depth = 0) {
2597 // Return false if we have an incomplete count.
2599 return false;
2600
2601 auto *I = dyn_cast<Instruction>(V);
2602 if (I == nullptr)
2603 return true;
2604
2605 if (Necessary != nullptr) {
2606 // This instruction is necessary for the other side of the condition so
2607 // don't count it.
2608 if (Necessary->contains(I))
2609 return true;
2610 }
2611
2612 // Already added this dep.
2613 if (!Deps->try_emplace(I, false).second)
2614 return true;
2615
2616 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2617 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2618 Depth + 1))
2619 return false;
2620 return true;
2621}
2622
2625 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2627 if (Params.BaseCost < 0)
2628 return false;
2629
2630 // Baseline cost.
2631 InstructionCost CostThresh = Params.BaseCost;
2632
2633 BranchProbabilityInfo *BPI = nullptr;
2634 if (Params.LikelyBias || Params.UnlikelyBias)
2635 BPI = FuncInfo.BPI;
2636 if (BPI != nullptr) {
2637 // See if we are either likely to get an early out or compute both lhs/rhs
2638 // of the condition.
2639 BasicBlock *IfTrue = I.getSuccessor(0);
2640 BasicBlock *IfFalse = I.getSuccessor(1);
2642 std::swap(IfTrue, IfFalse);
2643
2644 std::optional<bool> Likely;
2645 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2646 Likely = true;
2647 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2648 Likely = false;
2649
2650 if (Likely) {
2651 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2652 // Its likely we will have to compute both lhs and rhs of condition
2653 CostThresh += Params.LikelyBias;
2654 else {
2655 if (Params.UnlikelyBias < 0)
2656 return false;
2657 // Its likely we will get an early out.
2658 CostThresh -= Params.UnlikelyBias;
2659 }
2660 }
2661 }
2662
2663 if (CostThresh <= 0)
2664 return false;
2665
2666 // Collect "all" instructions that lhs condition is dependent on.
2667 // Use map for stable iteration (to avoid non-determanism of iteration of
2668 // SmallPtrSet). The `bool` value is just a dummy.
2670 collectInstructionDeps(&LhsDeps, Lhs);
2671 // Collect "all" instructions that rhs condition is dependent on AND are
2672 // dependencies of lhs. This gives us an estimate on which instructions we
2673 // stand to save by splitting the condition.
2674 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2675 return false;
2676 // Add the compare instruction itself unless its a dependency on the LHS.
2677 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2678 if (!LhsDeps.contains(RhsI))
2679 RhsDeps.try_emplace(RhsI, false);
2680
2681 InstructionCost CostOfIncluding = 0;
2682 // See if this instruction will need to computed independently of whether RHS
2683 // is.
2684 Value *BrCond = I.getCondition();
2685 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2686 for (const auto *U : Ins->users()) {
2687 // If user is independent of RHS calculation we don't need to count it.
2688 if (auto *UIns = dyn_cast<Instruction>(U))
2689 if (UIns != BrCond && !RhsDeps.contains(UIns))
2690 return false;
2691 }
2692 return true;
2693 };
2694
2695 // Prune instructions from RHS Deps that are dependencies of unrelated
2696 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2697 // arbitrary and just meant to cap the how much time we spend in the pruning
2698 // loop. Its highly unlikely to come into affect.
2699 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2700 // Stop after a certain point. No incorrectness from including too many
2701 // instructions.
2702 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2703 const Instruction *ToDrop = nullptr;
2704 for (const auto &InsPair : RhsDeps) {
2705 if (!ShouldCountInsn(InsPair.first)) {
2706 ToDrop = InsPair.first;
2707 break;
2708 }
2709 }
2710 if (ToDrop == nullptr)
2711 break;
2712 RhsDeps.erase(ToDrop);
2713 }
2714
2715 for (const auto &InsPair : RhsDeps) {
2716 // Finally accumulate latency that we can only attribute to computing the
2717 // RHS condition. Use latency because we are essentially trying to calculate
2718 // the cost of the dependency chain.
2719 // Possible TODO: We could try to estimate ILP and make this more precise.
2720 CostOfIncluding += TTI->getInstructionCost(
2721 InsPair.first, TargetTransformInfo::TCK_Latency);
2722
2723 if (CostOfIncluding > CostThresh)
2724 return false;
2725 }
2726 return true;
2727}
2728
2731 MachineBasicBlock *FBB,
2732 MachineBasicBlock *CurBB,
2733 MachineBasicBlock *SwitchBB,
2735 BranchProbability TProb,
2736 BranchProbability FProb,
2737 bool InvertCond) {
2738 // Skip over not part of the tree and remember to invert op and operands at
2739 // next level.
2740 Value *NotCond;
2741 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2742 InBlock(NotCond, CurBB->getBasicBlock())) {
2743 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2744 !InvertCond);
2745 return;
2746 }
2747
2749 const Value *BOpOp0, *BOpOp1;
2750 // Compute the effective opcode for Cond, taking into account whether it needs
2751 // to be inverted, e.g.
2752 // and (not (or A, B)), C
2753 // gets lowered as
2754 // and (and (not A, not B), C)
2756 if (BOp) {
2757 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2758 ? Instruction::And
2759 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2760 ? Instruction::Or
2762 if (InvertCond) {
2763 if (BOpc == Instruction::And)
2764 BOpc = Instruction::Or;
2765 else if (BOpc == Instruction::Or)
2766 BOpc = Instruction::And;
2767 }
2768 }
2769
2770 // If this node is not part of the or/and tree, emit it as a branch.
2771 // Note that all nodes in the tree should have same opcode.
2772 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2773 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2774 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2775 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2776 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2777 TProb, FProb, InvertCond);
2778 return;
2779 }
2780
2781 // Create TmpBB after CurBB.
2782 MachineFunction::iterator BBI(CurBB);
2783 MachineFunction &MF = DAG.getMachineFunction();
2785 CurBB->getParent()->insert(++BBI, TmpBB);
2786
2787 if (Opc == Instruction::Or) {
2788 // Codegen X | Y as:
2789 // BB1:
2790 // jmp_if_X TBB
2791 // jmp TmpBB
2792 // TmpBB:
2793 // jmp_if_Y TBB
2794 // jmp FBB
2795 //
2796
2797 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2798 // The requirement is that
2799 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2800 // = TrueProb for original BB.
2801 // Assuming the original probabilities are A and B, one choice is to set
2802 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2803 // A/(1+B) and 2B/(1+B). This choice assumes that
2804 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2805 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2806 // TmpBB, but the math is more complicated.
2807
2808 auto NewTrueProb = TProb / 2;
2809 auto NewFalseProb = TProb / 2 + FProb;
2810 // Emit the LHS condition.
2811 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2812 NewFalseProb, InvertCond);
2813
2814 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2815 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2817 // Emit the RHS condition into TmpBB.
2818 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2819 Probs[1], InvertCond);
2820 } else {
2821 assert(Opc == Instruction::And && "Unknown merge op!");
2822 // Codegen X & Y as:
2823 // BB1:
2824 // jmp_if_X TmpBB
2825 // jmp FBB
2826 // TmpBB:
2827 // jmp_if_Y TBB
2828 // jmp FBB
2829 //
2830 // This requires creation of TmpBB after CurBB.
2831
2832 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2833 // The requirement is that
2834 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2835 // = FalseProb for original BB.
2836 // Assuming the original probabilities are A and B, one choice is to set
2837 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2838 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2839 // TrueProb for BB1 * FalseProb for TmpBB.
2840
2841 auto NewTrueProb = TProb + FProb / 2;
2842 auto NewFalseProb = FProb / 2;
2843 // Emit the LHS condition.
2844 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2845 NewFalseProb, InvertCond);
2846
2847 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2848 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2850 // Emit the RHS condition into TmpBB.
2851 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2852 Probs[1], InvertCond);
2853 }
2854}
2855
2856/// If the set of cases should be emitted as a series of branches, return true.
2857/// If we should emit this as a bunch of and/or'd together conditions, return
2858/// false.
2859bool
2860SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2861 if (Cases.size() != 2) return true;
2862
2863 // If this is two comparisons of the same values or'd or and'd together, they
2864 // will get folded into a single comparison, so don't emit two blocks.
2865 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2866 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2867 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2868 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2869 return false;
2870 }
2871
2872 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2873 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2874 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2875 Cases[0].CC == Cases[1].CC &&
2876 isa<Constant>(Cases[0].CmpRHS) &&
2877 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2878 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2879 return false;
2880 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2881 return false;
2882 }
2883
2884 return true;
2885}
2886
2887void SelectionDAGBuilder::visitUncondBr(const UncondBrInst &I) {
2889
2890 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2891
2892 // Update machine-CFG edges.
2893 BrMBB->addSuccessor(Succ0MBB);
2894
2895 // If this is not a fall-through branch or optimizations are switched off,
2896 // emit the branch.
2897 if (Succ0MBB != NextBlock(BrMBB) ||
2899 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2900 DAG.getBasicBlock(Succ0MBB));
2901 setValue(&I, Br);
2902 DAG.setRoot(Br);
2903 }
2904}
2905
2906void SelectionDAGBuilder::visitCondBr(const CondBrInst &I) {
2907 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2908
2909 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2910
2911 // If this condition is one of the special cases we handle, do special stuff
2912 // now.
2913 const Value *CondVal = I.getCondition();
2914 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(I.getSuccessor(1));
2915
2916 // If this is a series of conditions that are or'd or and'd together, emit
2917 // this as a sequence of branches instead of setcc's with and/or operations.
2918 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2919 // unpredictable branches, and vector extracts because those jumps are likely
2920 // expensive for any target), this should improve performance.
2921 // For example, instead of something like:
2922 // cmp A, B
2923 // C = seteq
2924 // cmp D, E
2925 // F = setle
2926 // or C, F
2927 // jnz foo
2928 // Emit:
2929 // cmp A, B
2930 // je foo
2931 // cmp D, E
2932 // jle foo
2933 bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
2934 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2935 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2936 BOp->hasOneUse() && !IsUnpredictable) {
2937 Value *Vec;
2938 const Value *BOp0, *BOp1;
2940 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2941 Opcode = Instruction::And;
2942 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2943 Opcode = Instruction::Or;
2944
2945 if (Opcode &&
2946 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2947 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2949 FuncInfo, I, Opcode, BOp0, BOp1,
2950 DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2951 Opcode, BOp0, BOp1, FuncInfo.Fn))) {
2952 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2953 getEdgeProbability(BrMBB, Succ0MBB),
2954 getEdgeProbability(BrMBB, Succ1MBB),
2955 /*InvertCond=*/false);
2956 // If the compares in later blocks need to use values not currently
2957 // exported from this block, export them now. This block should always
2958 // be the first entry.
2959 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2960
2961 // Allow some cases to be rejected.
2962 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2963 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2964 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2965 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2966 }
2967
2968 // Emit the branch for this block.
2969 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2970 SL->SwitchCases.erase(SL->SwitchCases.begin());
2971 return;
2972 }
2973
2974 // Okay, we decided not to do this, remove any inserted MBB's and clear
2975 // SwitchCases.
2976 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2977 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2978
2979 SL->SwitchCases.clear();
2980 }
2981 }
2982
2983 // Create a CaseBlock record representing this branch.
2984 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2985 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2987 IsUnpredictable);
2988
2989 // Use visitSwitchCase to actually insert the fast branch sequence for this
2990 // cond branch.
2991 visitSwitchCase(CB, BrMBB);
2992}
2993
2994/// visitSwitchCase - Emits the necessary code to represent a single node in
2995/// the binary search tree resulting from lowering a switch instruction.
2997 MachineBasicBlock *SwitchBB) {
2998 SDValue Cond;
2999 SDValue CondLHS = getValue(CB.CmpLHS);
3000 SDLoc dl = CB.DL;
3001
3002 if (CB.CC == ISD::SETTRUE) {
3003 // Branch or fall through to TrueBB.
3004 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
3005 SwitchBB->normalizeSuccProbs();
3006 if (CB.TrueBB != NextBlock(SwitchBB)) {
3007 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
3008 DAG.getBasicBlock(CB.TrueBB)));
3009 }
3010 return;
3011 }
3012
3013 auto &TLI = DAG.getTargetLoweringInfo();
3014 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
3015
3016 // Build the setcc now.
3017 if (!CB.CmpMHS) {
3018 // Fold "(X == true)" to X and "(X == false)" to !X to
3019 // handle common cases produced by branch lowering.
3020 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
3021 CB.CC == ISD::SETEQ)
3022 Cond = CondLHS;
3023 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
3024 CB.CC == ISD::SETEQ) {
3025 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
3026 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
3027 } else {
3028 SDValue CondRHS = getValue(CB.CmpRHS);
3029
3030 // If a pointer's DAG type is larger than its memory type then the DAG
3031 // values are zero-extended. This breaks signed comparisons so truncate
3032 // back to the underlying type before doing the compare.
3033 if (CondLHS.getValueType() != MemVT) {
3034 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
3035 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
3036 }
3037 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
3038 }
3039 } else {
3040 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
3041
3042 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
3043 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
3044
3045 SDValue CmpOp = getValue(CB.CmpMHS);
3046 EVT VT = CmpOp.getValueType();
3047
3048 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
3049 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
3050 ISD::SETLE);
3051 } else {
3052 SDValue SUB = DAG.getNode(ISD::SUB, dl,
3053 VT, CmpOp, DAG.getConstant(Low, dl, VT));
3054 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
3055 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
3056 }
3057 }
3058
3059 // Update successor info
3060 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
3061 // TrueBB and FalseBB are always different unless the incoming IR is
3062 // degenerate. This only happens when running llc on weird IR.
3063 if (CB.TrueBB != CB.FalseBB)
3064 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
3065 SwitchBB->normalizeSuccProbs();
3066
3067 // If the lhs block is the next block, invert the condition so that we can
3068 // fall through to the lhs instead of the rhs block.
3069 if (CB.TrueBB == NextBlock(SwitchBB)) {
3070 std::swap(CB.TrueBB, CB.FalseBB);
3071 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
3072 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
3073 }
3074
3075 SDNodeFlags Flags;
3077 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
3078 Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
3079
3080 setValue(CurInst, BrCond);
3081
3082 // Insert the false branch. Do this even if it's a fall through branch,
3083 // this makes it easier to do DAG optimizations which require inverting
3084 // the branch condition.
3085 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3086 DAG.getBasicBlock(CB.FalseBB));
3087
3088 DAG.setRoot(BrCond);
3089}
3090
3091/// visitJumpTable - Emit JumpTable node in the current MBB
3093 // Emit the code for the jump table
3094 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3095 assert(JT.Reg && "Should lower JT Header first!");
3096 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DAG.getDataLayout());
3097 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
3098 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
3099 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
3100 Index.getValue(1), Table, Index);
3101 DAG.setRoot(BrJumpTable);
3102}
3103
3104/// visitJumpTableHeader - This function emits necessary code to produce index
3105/// in the JumpTable from switch case.
3107 JumpTableHeader &JTH,
3108 MachineBasicBlock *SwitchBB) {
3109 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3110 const SDLoc &dl = *JT.SL;
3111
3112 // Subtract the lowest switch case value from the value being switched on.
3113 SDValue SwitchOp = getValue(JTH.SValue);
3114 EVT VT = SwitchOp.getValueType();
3115 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
3116 DAG.getConstant(JTH.First, dl, VT));
3117
3118 // The SDNode we just created, which holds the value being switched on minus
3119 // the smallest case value, needs to be copied to a virtual register so it
3120 // can be used as an index into the jump table in a subsequent basic block.
3121 // This value may be smaller or larger than the target's pointer type, and
3122 // therefore require extension or truncating.
3123 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3124 SwitchOp =
3125 DAG.getZExtOrTrunc(Sub, dl, TLI.getJumpTableRegTy(DAG.getDataLayout()));
3126
3127 Register JumpTableReg =
3128 FuncInfo.CreateReg(TLI.getJumpTableRegTy(DAG.getDataLayout()));
3129 SDValue CopyTo =
3130 DAG.getCopyToReg(getControlRoot(), dl, JumpTableReg, SwitchOp);
3131 JT.Reg = JumpTableReg;
3132
3133 if (!JTH.FallthroughUnreachable) {
3134 // Emit the range check for the jump table, and branch to the default block
3135 // for the switch statement if the value being switched on exceeds the
3136 // largest case in the switch.
3137 SDValue CMP = DAG.getSetCC(
3138 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3139 Sub.getValueType()),
3140 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3141
3142 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3143 MVT::Other, CopyTo, CMP,
3144 DAG.getBasicBlock(JT.Default));
3145
3146 // Avoid emitting unnecessary branches to the next block.
3147 if (JT.MBB != NextBlock(SwitchBB))
3148 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3149 DAG.getBasicBlock(JT.MBB));
3150
3151 DAG.setRoot(BrCond);
3152 } else {
3153 // Avoid emitting unnecessary branches to the next block.
3154 if (JT.MBB != NextBlock(SwitchBB))
3155 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3156 DAG.getBasicBlock(JT.MBB)));
3157 else
3158 DAG.setRoot(CopyTo);
3159 }
3160}
3161
3162/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3163/// variable if there exists one.
3165 SDValue &Chain) {
3167 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3168 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3170 Value *Global =
3171 TLI.getSDagStackGuard(*MF.getFunction().getParent(), DAG.getLibcalls());
3173 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3174 if (Global) {
3175 MachinePointerInfo MPInfo(Global);
3179 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
3180 DAG.setNodeMemRefs(Node, {MemRef});
3181 }
3182 if (PtrTy != PtrMemTy)
3183 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3184 return SDValue(Node, 0);
3185}
3186
3187/// Codegen a new tail for a stack protector check ParentMBB which has had its
3188/// tail spliced into a stack protector check success bb.
3189///
3190/// For a high level explanation of how this fits into the stack protector
3191/// generation see the comment on the declaration of class
3192/// StackProtectorDescriptor.
3194 MachineBasicBlock *ParentBB) {
3195
3196 // First create the loads to the guard/stack slot for the comparison.
3197 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3198 auto &DL = DAG.getDataLayout();
3199 EVT PtrTy = TLI.getFrameIndexTy(DL);
3200 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
3201
3202 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3203 int FI = MFI.getStackProtectorIndex();
3204
3205 SDValue Guard;
3206 SDLoc dl = getCurSDLoc();
3207 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3208 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3209 Align Align = DL.getPrefTypeAlign(
3210 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
3211
3212 // Generate code to load the content of the guard slot.
3213 SDValue GuardVal = DAG.getLoad(
3214 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3215 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
3217
3218 // If cookie mixing is enabled, unmix the stored GuardVal to get back the
3219 // original cookie for comparison. The prologue stored (FP - Cookie) or
3220 // (FP XOR Cookie), so we apply the same operation again to unmix:
3221 // FP - (FP - Cookie) = Cookie, or (FP XOR Cookie) XOR FP = Cookie.
3222 if (TLI.useStackGuardMixFP())
3223 GuardVal = TLI.emitStackGuardMixFP(DAG, GuardVal, dl);
3224
3225 // If we're using function-based instrumentation, call the guard check
3226 // function
3228 // Get the guard check function from the target and verify it exists since
3229 // we're using function-based instrumentation
3230 const Function *GuardCheckFn =
3231 TLI.getSSPStackGuardCheck(M, DAG.getLibcalls());
3232 assert(GuardCheckFn && "Guard check function is null");
3233
3234 // The target provides a guard check function to validate the guard value.
3235 // Generate a call to that function with the content of the guard slot as
3236 // argument.
3237 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3238 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3239
3241 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
3242 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3243 Entry.IsInReg = true;
3244 Args.push_back(Entry);
3245
3248 .setChain(DAG.getEntryNode())
3249 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3250 getValue(GuardCheckFn), std::move(Args));
3251
3252 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3253 DAG.setRoot(Result.second);
3254 return;
3255 }
3256
3257 // Load the fresh guard value for comparison.
3258 // For targets that mix the cookie in LOAD_STACK_GUARD expansion, we need to
3259 // load directly without using LOAD_STACK_GUARD to avoid unwanted mixing.
3260 SDValue Chain = DAG.getEntryNode();
3261 if (TLI.useStackGuardMixFP()) {
3262 // Mixing targets: load cookie directly to avoid mixing in LOAD_STACK_GUARD
3263 if (const Value *IRGuard = TLI.getSDagStackGuard(M, DAG.getLibcalls())) {
3264 SDValue GuardPtr = getValue(IRGuard);
3265 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3266 MachinePointerInfo(IRGuard, 0), Align,
3268 } else {
3269 LLVMContext &Ctx = *DAG.getContext();
3270 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
3271 Guard = DAG.getPOISON(PtrMemTy);
3272 }
3273 } else {
3274 // Non-mixing targets: use LOAD_STACK_GUARD or direct load as usual
3275 if (TLI.useLoadStackGuardNode(M)) {
3276 Guard = getLoadStackGuard(DAG, dl, Chain);
3277 } else {
3278 if (const Value *IRGuard = TLI.getSDagStackGuard(M, DAG.getLibcalls())) {
3279 SDValue GuardPtr = getValue(IRGuard);
3280 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3281 MachinePointerInfo(IRGuard, 0), Align,
3283 } else {
3284 LLVMContext &Ctx = *DAG.getContext();
3285 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
3286 Guard = DAG.getPOISON(PtrMemTy);
3287 }
3288 }
3289 }
3290
3291 // Now both Guard (fresh cookie) and GuardVal (unmixed from stored value)
3292 // contain unmixed cookie values that can be compared directly.
3293
3294 // Perform the comparison via a getsetcc.
3295 SDValue Cmp = DAG.getSetCC(
3296 dl, TLI.getSetCCResultType(DL, *DAG.getContext(), Guard.getValueType()),
3297 Guard, GuardVal, ISD::SETNE);
3298
3299 // If the guard/stackslot do not equal, branch to failure MBB.
3300 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
3301 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3302 // Otherwise branch to success MBB.
3303 SDValue Br = DAG.getNode(ISD::BR, dl,
3304 MVT::Other, BrCond,
3305 DAG.getBasicBlock(SPD.getSuccessMBB()));
3306
3307 DAG.setRoot(Br);
3308}
3309
3310/// Codegen the failure basic block for a stack protector check.
3311///
3312/// A failure stack protector machine basic block consists simply of a call to
3313/// __stack_chk_fail().
3314///
3315/// For a high level explanation of how this fits into the stack protector
3316/// generation see the comment on the declaration of class
3317/// StackProtectorDescriptor.
3320
3321 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3322 MachineBasicBlock *ParentBB = SPD.getParentMBB();
3323 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3324 SDValue Chain;
3325
3326 // For -Oz builds with a guard check function, we use function-based
3327 // instrumentation. Otherwise, if we have a guard check function, we call it
3328 // in the failure block.
3329 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M, DAG.getLibcalls());
3330 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3331 // First create the loads to the guard/stack slot for the comparison.
3332 auto &DL = DAG.getDataLayout();
3333 EVT PtrTy = TLI.getFrameIndexTy(DL);
3334 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
3335
3336 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3337 int FI = MFI.getStackProtectorIndex();
3338
3339 SDLoc dl = getCurSDLoc();
3340 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3341 Align Align = DL.getPrefTypeAlign(
3342 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
3343
3344 // Generate code to load the content of the guard slot.
3345 SDValue GuardVal = DAG.getLoad(
3346 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3347 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
3349
3350 if (TLI.useStackGuardMixFP())
3351 GuardVal = TLI.emitStackGuardMixFP(DAG, GuardVal, dl);
3352
3353 // The target provides a guard check function to validate the guard value.
3354 // Generate a call to that function with the content of the guard slot as
3355 // argument.
3356 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3357 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3358
3360 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
3361 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3362 Entry.IsInReg = true;
3363 Args.push_back(Entry);
3364
3367 .setChain(DAG.getEntryNode())
3368 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3369 getValue(GuardCheckFn), std::move(Args));
3370
3371 Chain = TLI.LowerCallTo(CLI).second;
3372 } else {
3374 CallOptions.setDiscardResult(true);
3375 Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3376 {}, CallOptions, getCurSDLoc())
3377 .second;
3378 }
3379
3380 // Emit a trap instruction if we are required to do so.
3381 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3382 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3383 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3384
3385 DAG.setRoot(Chain);
3386}
3387
3388/// visitBitTestHeader - This function emits necessary code to produce value
3389/// suitable for "bit tests"
3391 MachineBasicBlock *SwitchBB) {
3392 SDLoc dl = getCurSDLoc();
3393
3394 // Subtract the minimum value.
3395 SDValue SwitchOp = getValue(B.SValue);
3396 EVT VT = SwitchOp.getValueType();
3397 SDValue RangeSub =
3398 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3399
3400 // Determine the type of the test operands.
3401 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3402 bool UsePtrType = false;
3403 if (!TLI.isTypeLegal(VT)) {
3404 UsePtrType = true;
3405 } else {
3406 for (const BitTestCase &Case : B.Cases)
3407 if (!isUIntN(VT.getSizeInBits(), Case.Mask)) {
3408 // Switch table case range are encoded into series of masks.
3409 // Just use pointer type, it's guaranteed to fit.
3410 UsePtrType = true;
3411 break;
3412 }
3413 }
3414 SDValue Sub = RangeSub;
3415 if (UsePtrType) {
3416 VT = TLI.getPointerTy(DAG.getDataLayout());
3417 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3418 }
3419
3420 B.RegVT = VT.getSimpleVT();
3421 B.Reg = FuncInfo.CreateReg(B.RegVT);
3422 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3423
3424 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3425
3426 if (!B.FallthroughUnreachable)
3427 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3428 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3429 SwitchBB->normalizeSuccProbs();
3430
3431 SDValue Root = CopyTo;
3432 if (!B.FallthroughUnreachable) {
3433 // Conditional branch to the default block.
3434 SDValue RangeCmp = DAG.getSetCC(dl,
3435 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3436 RangeSub.getValueType()),
3437 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3438 ISD::SETUGT);
3439
3440 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3441 DAG.getBasicBlock(B.Default));
3442 }
3443
3444 // Avoid emitting unnecessary branches to the next block.
3445 if (MBB != NextBlock(SwitchBB))
3446 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3447
3448 DAG.setRoot(Root);
3449}
3450
3451/// visitBitTestCase - this function produces one "bit test"
3453 MachineBasicBlock *NextMBB,
3454 BranchProbability BranchProbToNext,
3455 Register Reg, BitTestCase &B,
3456 MachineBasicBlock *SwitchBB) {
3457 SDLoc dl = getCurSDLoc();
3458 MVT VT = BB.RegVT;
3459 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3460 SDValue Cmp;
3461 unsigned PopCount = llvm::popcount(B.Mask);
3462 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3463 if (PopCount == 1) {
3464 // Testing for a single bit; just compare the shift count with what it
3465 // would need to be to shift a 1 bit in that position.
3466 Cmp = DAG.getSetCC(
3467 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3468 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3469 ISD::SETEQ);
3470 } else if (PopCount == BB.Range) {
3471 // There is only one zero bit in the range, test for it directly.
3472 Cmp = DAG.getSetCC(
3473 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3474 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3475 } else {
3476 // Make desired shift
3477 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3478 DAG.getConstant(1, dl, VT), ShiftOp);
3479
3480 // Emit bit tests and jumps
3481 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3482 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3483 Cmp = DAG.getSetCC(
3484 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3485 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3486 }
3487
3488 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3489 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3490 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3491 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3492 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3493 // one as they are relative probabilities (and thus work more like weights),
3494 // and hence we need to normalize them to let the sum of them become one.
3495 SwitchBB->normalizeSuccProbs();
3496
3497 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3498 MVT::Other, getControlRoot(),
3499 Cmp, DAG.getBasicBlock(B.TargetBB));
3500
3501 // Avoid emitting unnecessary branches to the next block.
3502 if (NextMBB != NextBlock(SwitchBB))
3503 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3504 DAG.getBasicBlock(NextMBB));
3505
3506 DAG.setRoot(BrAnd);
3507}
3508
3509void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3510 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3511
3512 // Retrieve successors. Look through artificial IR level blocks like
3513 // catchswitch for successors.
3514 MachineBasicBlock *Return = FuncInfo.getMBB(I.getSuccessor(0));
3515 const BasicBlock *EHPadBB = I.getSuccessor(1);
3516 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(EHPadBB);
3517
3518 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3519 // have to do anything here to lower funclet bundles.
3520 failForInvalidBundles(I, "invokes",
3526
3527 const Value *Callee(I.getCalledOperand());
3528 const Function *Fn = dyn_cast<Function>(Callee);
3529 if (isa<InlineAsm>(Callee))
3530 visitInlineAsm(I, EHPadBB);
3531 else if (Fn && Fn->isIntrinsic()) {
3532 switch (Fn->getIntrinsicID()) {
3533 default:
3534 llvm_unreachable("Cannot invoke this intrinsic");
3535 case Intrinsic::donothing:
3536 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3537 case Intrinsic::seh_try_begin:
3538 case Intrinsic::seh_scope_begin:
3539 case Intrinsic::seh_try_end:
3540 case Intrinsic::seh_scope_end:
3541 if (EHPadMBB)
3542 // a block referenced by EH table
3543 // so dtor-funclet not removed by opts
3544 EHPadMBB->setMachineBlockAddressTaken();
3545 break;
3546 case Intrinsic::experimental_patchpoint_void:
3547 case Intrinsic::experimental_patchpoint:
3548 visitPatchpoint(I, EHPadBB);
3549 break;
3550 case Intrinsic::experimental_gc_statepoint:
3552 break;
3553 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3554 // but these intrinsics are special because they can be invoked, so we
3555 // manually lower it to a DAG node here.
3556 case Intrinsic::wasm_throw: {
3558 std::array<SDValue, 4> Ops = {
3559 getControlRoot(), // inchain for the terminator node
3560 DAG.getTargetConstant(Intrinsic::wasm_throw, getCurSDLoc(),
3561 TLI.getPointerTy(DAG.getDataLayout())),
3562 getValue(I.getArgOperand(0)), // tag
3563 getValue(I.getArgOperand(1)) // thrown value
3564 };
3565 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3566 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3567 break;
3568 }
3569 case Intrinsic::wasm_rethrow: {
3570 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3571 std::array<SDValue, 2> Ops = {
3572 getControlRoot(), // inchain for the terminator node
3573 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3574 TLI.getPointerTy(DAG.getDataLayout()))};
3575 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3576 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3577 break;
3578 }
3579 // EH labels mark it as an invoke site, like a call.
3580 case Intrinsic::bpf_unwind_exit: {
3581 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3582 MCSymbol *BeginLabel;
3583 std::array<SDValue, 2> Ops = {
3584 lowerStartEH(getControlRoot(), EHPadBB, BeginLabel),
3585 DAG.getTargetConstant(Intrinsic::bpf_unwind_exit, getCurSDLoc(),
3586 TLI.getPointerTy(DAG.getDataLayout()))};
3587 SDValue Chain =
3588 DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), MVT::Other, Ops);
3589 DAG.setRoot(lowerEndEH(Chain, &I, EHPadBB, BeginLabel));
3590 break;
3591 }
3592 }
3593 } else if (I.hasDeoptState()) {
3594 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3595 // Eventually we will support lowering the @llvm.experimental.deoptimize
3596 // intrinsic, and right now there are no plans to support other intrinsics
3597 // with deopt state.
3598 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3599 } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
3601 } else {
3602 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3603 }
3604
3605 // If the value of the invoke is used outside of its defining block, make it
3606 // available as a virtual register.
3607 // We already took care of the exported value for the statepoint instruction
3608 // during call to the LowerStatepoint.
3609 if (!isa<GCStatepointInst>(I)) {
3611 }
3612
3614 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3615 BranchProbability EHPadBBProb =
3616 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3618 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3619
3620 // Update successor info.
3621 addSuccessorWithProb(InvokeMBB, Return);
3622 for (auto &UnwindDest : UnwindDests) {
3623 UnwindDest.first->setIsEHPad();
3624 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3625 }
3626 InvokeMBB->normalizeSuccProbs();
3627
3628 // Drop into normal successor.
3629 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
3630 DAG.getBasicBlock(Return)));
3631}
3632
3633/// The intrinsics currently supported by callbr are implicit control flow
3634/// intrinsics such as amdgcn.kill.
3635/// - they should be called (no "dontcall-" attributes)
3636/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
3637/// - they do not need custom argument handling (no
3638/// TLI.CollectTargetIntrinsicOperands())
3639void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
3640#ifndef NDEBUG
3642 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3643 Infos, I, DAG.getMachineFunction(), I.getIntrinsicID());
3644 assert(Infos.empty() && "Intrinsic touches memory");
3645#endif
3646
3647 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
3648
3650 getTargetIntrinsicOperands(I, HasChain, OnlyLoad);
3651 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
3652
3653 // Create the node.
3654 SDValue Result =
3655 getTargetNonMemIntrinsicNode(*I.getType(), HasChain, Ops, VTs);
3656 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
3657
3658 setValue(&I, Result);
3659}
3660
3661void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3662 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3663
3664 if (I.isInlineAsm()) {
3665 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3666 // have to do anything here to lower funclet bundles.
3667 failForInvalidBundles(I, "callbrs",
3669 visitInlineAsm(I);
3670 } else {
3671 assert(!I.hasOperandBundles() &&
3672 "Can't have operand bundles for intrinsics");
3673 visitCallBrIntrinsic(I);
3674 }
3676
3677 // Retrieve successors.
3678 SmallPtrSet<BasicBlock *, 8> Dests;
3679 Dests.insert(I.getDefaultDest());
3680 MachineBasicBlock *Return = FuncInfo.getMBB(I.getDefaultDest());
3681
3682 // Update successor info.
3683 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3684 // TODO: For most of the cases where there is an intrinsic callbr, we're
3685 // having exactly one indirect target, which will be unreachable. As soon as
3686 // this changes, we might need to enhance
3687 // Target->setIsInlineAsmBrIndirectTarget or add something similar for
3688 // intrinsic indirect branches.
3689 if (I.isInlineAsm()) {
3690 for (BasicBlock *Dest : I.getIndirectDests()) {
3691 MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
3692 Target->setIsInlineAsmBrIndirectTarget();
3693 // If we introduce a type of asm goto statement that is permitted to use
3694 // an indirect call instruction to jump to its labels, then we should add
3695 // a call to Target->setMachineBlockAddressTaken() here, to mark the
3696 // target block as requiring a BTI.
3697
3698 Target->setLabelMustBeEmitted();
3699 // Don't add duplicate machine successors.
3700 if (Dests.insert(Dest).second)
3701 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3702 }
3703 }
3704 CallBrMBB->normalizeSuccProbs();
3705
3706 // Drop into default successor.
3707 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3708 MVT::Other, getControlRoot(),
3709 DAG.getBasicBlock(Return)));
3710}
3711
3712void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3713 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3714}
3715
3716void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3717 assert(FuncInfo.MBB->isEHPad() &&
3718 "Call to landingpad not in landing pad!");
3719
3720 // If there aren't registers to copy the values into (e.g., during SjLj
3721 // exceptions), then don't bother to create these DAG nodes.
3722 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3723 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3724 if (TLI.getExceptionPointerRegister(FuncInfo.ExceptionModel, PersonalityFn) ==
3725 0 &&
3727 PersonalityFn) == 0)
3728 return;
3729
3730 // If landingpad's return type is token type, we don't create DAG nodes
3731 // for its exception pointer and selector value. The extraction of exception
3732 // pointer or selector value from token type landingpads is not currently
3733 // supported.
3734 if (LP.getType()->isTokenTy())
3735 return;
3736
3737 // LangRef leaves the result type target-specific, so diagnose types this
3738 // lowering cannot represent instead of asserting.
3739 SDLoc dl = getCurSDLoc();
3741 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
3742 *LP.getFunction(),
3743 "landingpad result type must be a struct of an exception pointer and "
3744 "an integer selector",
3745 dl.getDebugLoc()));
3746 setValueToPoison(&LP, dl);
3747 return;
3748 }
3749
3750 SmallVector<EVT, 2> ValueVTs;
3751 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3752 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3753
3754 // Get the two live-in registers as SDValues. The physregs have already been
3755 // copied into virtual registers.
3756 SDValue Ops[2];
3757 if (FuncInfo.ExceptionPointerVirtReg) {
3758 Ops[0] = DAG.getZExtOrTrunc(
3759 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3760 FuncInfo.ExceptionPointerVirtReg,
3761 TLI.getPointerTy(DAG.getDataLayout())),
3762 dl, ValueVTs[0]);
3763 } else {
3764 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3765 }
3766 Ops[1] = DAG.getZExtOrTrunc(
3767 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3768 FuncInfo.ExceptionSelectorVirtReg,
3769 TLI.getPointerTy(DAG.getDataLayout())),
3770 dl, ValueVTs[1]);
3771
3772 // Merge into one.
3773 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3774 DAG.getVTList(ValueVTs), Ops);
3775 setValue(&LP, Res);
3776}
3777
3780 // Update JTCases.
3781 for (JumpTableBlock &JTB : SL->JTCases)
3782 if (JTB.first.HeaderBB == First)
3783 JTB.first.HeaderBB = Last;
3784
3785 // Update BitTestCases.
3786 for (BitTestBlock &BTB : SL->BitTestCases)
3787 if (BTB.Parent == First)
3788 BTB.Parent = Last;
3789}
3790
3791void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3792 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3793
3794 // Update machine-CFG edges with unique successors.
3796 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3797 BasicBlock *BB = I.getSuccessor(i);
3798 bool Inserted = Done.insert(BB).second;
3799 if (!Inserted)
3800 continue;
3801
3802 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3803 addSuccessorWithProb(IndirectBrMBB, Succ);
3804 }
3805 IndirectBrMBB->normalizeSuccProbs();
3806
3808 MVT::Other, getControlRoot(),
3809 getValue(I.getAddress())));
3810}
3811
3812void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3813 if (!I.shouldLowerToTrap(DAG.getTarget().Options.TrapUnreachable,
3814 DAG.getTarget().Options.NoTrapAfterNoreturn))
3815 return;
3816
3817 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3818}
3819
3820void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3821 SDNodeFlags Flags;
3822 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3823 Flags.copyFMF(*FPOp);
3824
3825 SDValue Op = getValue(I.getOperand(0));
3826 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3827 Op, Flags);
3828 setValue(&I, UnNodeValue);
3829}
3830
3831void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3832 SDNodeFlags Flags;
3833 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3834 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3835 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3836 }
3837 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3838 Flags.setExact(ExactOp->isExact());
3839 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3840 Flags.setDisjoint(DisjointOp->isDisjoint());
3841 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3842 Flags.copyFMF(*FPOp);
3843
3844 SDValue Op1 = getValue(I.getOperand(0));
3845 SDValue Op2 = getValue(I.getOperand(1));
3846 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3847 Op1, Op2, Flags);
3848 setValue(&I, BinNodeValue);
3849}
3850
3851void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3852 SDValue Op1 = getValue(I.getOperand(0));
3853 SDValue Op2 = getValue(I.getOperand(1));
3854
3855 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3856 Op1.getValueType(), DAG.getDataLayout());
3857
3858 // Coerce the shift amount to the right type if we can. This exposes the
3859 // truncate or zext to optimization early.
3860 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3862 "Unexpected shift type");
3863 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3864 }
3865
3866 bool nuw = false;
3867 bool nsw = false;
3868 bool exact = false;
3869
3870 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3871
3872 if (const OverflowingBinaryOperator *OFBinOp =
3874 nuw = OFBinOp->hasNoUnsignedWrap();
3875 nsw = OFBinOp->hasNoSignedWrap();
3876 }
3877 if (const PossiblyExactOperator *ExactOp =
3879 exact = ExactOp->isExact();
3880 }
3881 SDNodeFlags Flags;
3882 Flags.setExact(exact);
3883 Flags.setNoSignedWrap(nsw);
3884 Flags.setNoUnsignedWrap(nuw);
3885 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3886 Flags);
3887 setValue(&I, Res);
3888}
3889
3890void SelectionDAGBuilder::visitSDiv(const User &I) {
3891 SDValue Op1 = getValue(I.getOperand(0));
3892 SDValue Op2 = getValue(I.getOperand(1));
3893
3894 SDNodeFlags Flags;
3895 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3896 cast<PossiblyExactOperator>(&I)->isExact());
3897 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3898 Op2, Flags));
3899}
3900
3901void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3902 ICmpInst::Predicate predicate = I.getPredicate();
3903 SDValue Op1 = getValue(I.getOperand(0));
3904 SDValue Op2 = getValue(I.getOperand(1));
3905 ISD::CondCode Opcode = getICmpCondCode(predicate);
3906
3907 auto &TLI = DAG.getTargetLoweringInfo();
3908 EVT MemVT =
3909 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3910
3911 // If a pointer's DAG type is larger than its memory type then the DAG values
3912 // are zero-extended. This breaks signed comparisons so truncate back to the
3913 // underlying type before doing the compare.
3914 if (Op1.getValueType() != MemVT) {
3915 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3916 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3917 }
3918
3919 SDNodeFlags Flags;
3920 Flags.setSameSign(I.hasSameSign());
3921
3922 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3923 I.getType());
3924 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode,
3925 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3926}
3927
3928void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3929 FCmpInst::Predicate predicate = I.getPredicate();
3930 SDValue Op1 = getValue(I.getOperand(0));
3931 SDValue Op2 = getValue(I.getOperand(1));
3932
3933 ISD::CondCode Condition = getFCmpCondCode(predicate);
3934 auto *FPMO = cast<FPMathOperator>(&I);
3935 if (FPMO->hasNoNaNs() ||
3936 (DAG.isKnownNeverNaN(Op1) && DAG.isKnownNeverNaN(Op2)))
3937 Condition = getFCmpCodeWithoutNaN(Condition);
3938
3939 SDNodeFlags Flags;
3940 Flags.copyFMF(*FPMO);
3941
3942 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3943 I.getType());
3944 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition,
3945 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3946}
3947
3948// Check if the condition of the select has one use or two users that are both
3949// selects with the same condition.
3950static bool hasOnlySelectUsers(const Value *Cond) {
3951 return llvm::all_of(Cond->users(), [](const Value *V) {
3952 return isa<SelectInst>(V);
3953 });
3954}
3955
3956void SelectionDAGBuilder::visitSelect(const User &I) {
3957 SmallVector<EVT, 4> ValueVTs;
3958 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3959 ValueVTs);
3960 unsigned NumValues = ValueVTs.size();
3961 if (NumValues == 0) return;
3962
3964 SDValue Cond = getValue(I.getOperand(0));
3965 SDValue LHSVal = getValue(I.getOperand(1));
3966 SDValue RHSVal = getValue(I.getOperand(2));
3967 SmallVector<SDValue, 1> BaseOps(1, Cond);
3969 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3970
3971 bool IsUnaryAbs = false;
3972 bool Negate = false;
3973
3974 SDNodeFlags Flags;
3975 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3976 Flags.copyFMF(*FPOp);
3977
3978 Flags.setUnpredictable(
3979 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3980
3981 // Min/max matching is only viable if all output VTs are the same.
3982 if (all_equal(ValueVTs)) {
3983 EVT VT = ValueVTs[0];
3984 LLVMContext &Ctx = *DAG.getContext();
3985 auto &TLI = DAG.getTargetLoweringInfo();
3986
3987 // We care about the legality of the operation after it has been type
3988 // legalized.
3990 VT = TLI.getTypeToTransformTo(Ctx, VT);
3991
3992 // If the vselect is legal, assume we want to leave this as a vector setcc +
3993 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3994 // min/max is legal on the scalar type.
3995 bool UseScalarMinMax = VT.isVector() &&
3997
3998 // ValueTracking's select pattern matching does not account for -0.0,
3999 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
4000 // -0.0 is less than +0.0.
4001 const Value *LHS, *RHS;
4002 auto SPR = matchSelectPattern(&I, LHS, RHS);
4004 switch (SPR.Flavor) {
4005 case SPF_UMAX: Opc = ISD::UMAX; break;
4006 case SPF_UMIN: Opc = ISD::UMIN; break;
4007 case SPF_SMAX: Opc = ISD::SMAX; break;
4008 case SPF_SMIN: Opc = ISD::SMIN; break;
4009 case SPF_FMINNUM:
4011 break;
4012
4013 switch (SPR.NaNBehavior) {
4014 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4015 case SPNB_RETURNS_ANY:
4016 case SPNB_RETURNS_NAN:
4017 break;
4018 case SPNB_RETURNS_OTHER:
4020 Flags.setNoSignedZeros(true);
4021 break;
4022 }
4023 break;
4024 case SPF_FMAXNUM:
4026 break;
4027
4028 switch (SPR.NaNBehavior) {
4029 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4030 case SPNB_RETURNS_NAN:
4031 case SPNB_RETURNS_ANY:
4032 break;
4033 case SPNB_RETURNS_OTHER:
4035 Flags.setNoSignedZeros(true);
4036 break;
4037 }
4038 break;
4039 case SPF_NABS:
4040 Negate = true;
4041 [[fallthrough]];
4042 case SPF_ABS:
4043 IsUnaryAbs = true;
4044 Opc = ISD::ABS;
4045 break;
4046 default: break;
4047 }
4048
4049 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
4051 (UseScalarMinMax &&
4053 // If the underlying comparison instruction is used by any other
4054 // instruction, the consumed instructions won't be destroyed, so it is
4055 // not profitable to convert to a min/max.
4057 OpCode = Opc;
4058 LHSVal = getValue(LHS);
4059 RHSVal = getValue(RHS);
4060 BaseOps.clear();
4061 }
4062
4063 if (IsUnaryAbs) {
4064 OpCode = Opc;
4065 LHSVal = getValue(LHS);
4066 BaseOps.clear();
4067 }
4068 }
4069
4070 if (IsUnaryAbs) {
4071 for (unsigned i = 0; i != NumValues; ++i) {
4072 SDLoc dl = getCurSDLoc();
4073 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
4074 Values[i] =
4075 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
4076 if (Negate)
4077 Values[i] = DAG.getNegative(Values[i], dl, VT);
4078 }
4079 } else {
4080 for (unsigned i = 0; i != NumValues; ++i) {
4081 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
4082 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
4083 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
4084 Values[i] = DAG.getNode(
4085 OpCode, getCurSDLoc(),
4086 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
4087 }
4088 }
4089
4091 DAG.getVTList(ValueVTs), Values));
4092}
4093
4094void SelectionDAGBuilder::visitTrunc(const User &I) {
4095 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
4096 SDValue N = getValue(I.getOperand(0));
4097 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4098 I.getType());
4099 SDNodeFlags Flags;
4100 if (auto *Trunc = dyn_cast<TruncInst>(&I)) {
4101 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
4102 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
4103 }
4104
4105 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N, Flags));
4106}
4107
4108void SelectionDAGBuilder::visitZExt(const User &I) {
4109 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4110 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
4111 SDValue N = getValue(I.getOperand(0));
4112 auto &TLI = DAG.getTargetLoweringInfo();
4113 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4114
4115 SDNodeFlags Flags;
4116 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
4117 Flags.setNonNeg(PNI->hasNonNeg());
4118
4119 // Eagerly use nonneg information to canonicalize towards sign_extend if
4120 // that is the target's preference.
4121 // TODO: Let the target do this later.
4122 if (Flags.hasNonNeg() &&
4123 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
4124 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
4125 return;
4126 }
4127
4128 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
4129}
4130
4131void SelectionDAGBuilder::visitSExt(const User &I) {
4132 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4133 // SExt also can't be a cast to bool for same reason. So, nothing much to do
4134 SDValue N = getValue(I.getOperand(0));
4135 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4136 I.getType());
4137 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
4138}
4139
4140void SelectionDAGBuilder::visitFPTrunc(const User &I) {
4141 // FPTrunc is never a no-op cast, no need to check
4142 SDValue N = getValue(I.getOperand(0));
4143 SDLoc dl = getCurSDLoc();
4144 SDNodeFlags Flags;
4145 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
4146 Flags.copyFMF(*FPOp);
4147 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4148 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4149 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
4150 DAG.getTargetConstant(
4151 0, dl, TLI.getPointerTy(DAG.getDataLayout())),
4152 Flags));
4153}
4154
4155void SelectionDAGBuilder::visitFPExt(const User &I) {
4156 // FPExt is never a no-op cast, no need to check
4157 SDValue N = getValue(I.getOperand(0));
4158 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4159 I.getType());
4160 SDNodeFlags Flags;
4161 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
4162 Flags.copyFMF(*FPOp);
4163 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N, Flags));
4164}
4165
4166void SelectionDAGBuilder::visitFPToUI(const User &I) {
4167 // FPToUI is never a no-op cast, no need to check
4168 SDValue N = getValue(I.getOperand(0));
4169 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4170 I.getType());
4171 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
4172}
4173
4174void SelectionDAGBuilder::visitFPToSI(const User &I) {
4175 // FPToSI is never a no-op cast, no need to check
4176 SDValue N = getValue(I.getOperand(0));
4177 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4178 I.getType());
4179 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
4180}
4181
4182void SelectionDAGBuilder::visitUIToFP(const User &I) {
4183 // UIToFP is never a no-op cast, no need to check
4184 SDValue N = getValue(I.getOperand(0));
4185 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4186 I.getType());
4187 SDNodeFlags Flags;
4188 Flags.setNonNeg(cast<PossiblyNonNegInst>(&I)->hasNonNeg());
4189 Flags.copyFMF(*cast<FPMathOperator>(&I));
4190
4191 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
4192}
4193
4194void SelectionDAGBuilder::visitSIToFP(const User &I) {
4195 // SIToFP is never a no-op cast, no need to check
4196 SDValue N = getValue(I.getOperand(0));
4197 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4198 I.getType());
4199 SDNodeFlags Flags;
4200 Flags.copyFMF(*cast<FPMathOperator>(&I));
4201
4202 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
4203}
4204
4205void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
4206 SDValue N = getValue(I.getOperand(0));
4207 // By definition the type of the ptrtoaddr must be equal to the address type.
4208 const auto &TLI = DAG.getTargetLoweringInfo();
4209 EVT AddrVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4210 // The address width must be smaller or equal to the pointer representation
4211 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).
4212 N = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), AddrVT, N);
4213 setValue(&I, N);
4214}
4215
4216void SelectionDAGBuilder::visitPtrToInt(const User &I) {
4217 // What to do depends on the size of the integer and the size of the pointer.
4218 // We can either truncate, zero extend, or no-op, accordingly.
4219 SDValue N = getValue(I.getOperand(0));
4220 auto &TLI = DAG.getTargetLoweringInfo();
4221 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4222 I.getType());
4223 EVT PtrMemVT =
4224 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
4225 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
4226 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
4227 setValue(&I, N);
4228}
4229
4230void SelectionDAGBuilder::visitIntToPtr(const User &I) {
4231 // What to do depends on the size of the integer and the size of the pointer.
4232 // We can either truncate, zero extend, or no-op, accordingly.
4233 SDValue N = getValue(I.getOperand(0));
4234 auto &TLI = DAG.getTargetLoweringInfo();
4235 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4236 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4237 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
4238 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
4239 setValue(&I, N);
4240}
4241
4242void SelectionDAGBuilder::visitBitCast(const User &I) {
4243 SDValue N = getValue(I.getOperand(0));
4244 SDLoc dl = getCurSDLoc();
4245 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4246 I.getType());
4247
4248 // BitCast assures us that source and destination are the same size so this is
4249 // either a BITCAST or a no-op.
4250 if (DestVT != N.getValueType())
4251 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
4252 DestVT, N)); // convert types.
4253 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
4254 // might fold any kind of constant expression to an integer constant and that
4255 // is not what we are looking for. Only recognize a bitcast of a genuine
4256 // constant integer as an opaque constant.
4257 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
4258 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
4259 /*isOpaque*/true));
4260 else
4261 setValue(&I, N); // noop cast.
4262}
4263
4264void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
4265 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4266 const Value *SV = I.getOperand(0);
4267 SDValue N = getValue(SV);
4268 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4269
4270 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
4271 unsigned DestAS = I.getType()->getPointerAddressSpace();
4272
4273 if (!TM.isNoopAddrSpaceCast(DAG.getDataLayout(), SrcAS, DestAS)) {
4274 SDNodeFlags Flags;
4275 if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(&I))
4276 Flags.setNonNull(ASC->hasNonNull());
4277 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS, Flags);
4278 }
4279
4280 setValue(&I, N);
4281}
4282
4283void SelectionDAGBuilder::visitInsertElement(const User &I) {
4284 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4285 SDValue InVec = getValue(I.getOperand(0));
4286 SDValue InVal = getValue(I.getOperand(1));
4287 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
4288 TLI.getVectorIdxTy(DAG.getDataLayout()));
4290 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4291 InVec, InVal, InIdx));
4292}
4293
4294void SelectionDAGBuilder::visitBitInsert(const User &I) {
4295 SDValue Base = getValue(I.getOperand(0));
4296 SDValue Val = getValue(I.getOperand(1));
4297 SDValue Offset = getValue(I.getOperand(2));
4298 EVT BaseVT = Base.getValueType();
4299 EVT ValVT = Val.getValueType();
4300 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4301 SDLoc dl = getCurSDLoc();
4302
4303 assert(BaseVT.getScalarSizeInBits() >= ValVT.getScalarSizeInBits() &&
4304 "bitinsert val wider than base should be rejected by verifier");
4305
4306 // If Val is a float, cast it to an integer of the same bitwidth
4307 // so DAG.getZExtOrTrunc can process it safely.
4308 if (!ValVT.isInteger()) {
4309 ValVT = ValVT.changeTypeToInteger();
4310 Val = DAG.getBitcast(ValVT, Val);
4311 }
4312
4313 // Legalize shift amount to the target's shift amount type.
4314 EVT ShiftAmtTy = TLI.getShiftAmountTy(BaseVT, DAG.getDataLayout());
4315 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Offset, dl, ShiftAmtTy);
4316
4317 unsigned BaseBitWidth = BaseVT.getScalarSizeInBits();
4318 unsigned ValBitWidth = ValVT.getScalarSizeInBits();
4319 APInt InsertMask = APInt::getLowBitsSet(BaseBitWidth, ValBitWidth);
4320 SDValue ShiftedMask =
4321 DAG.getNode(ISD::SHL, dl, BaseVT, DAG.getConstant(InsertMask, dl, BaseVT),
4322 LegalShiftAmount);
4323 SDValue ClearMask = DAG.getNOT(dl, ShiftedMask, BaseVT);
4324 SDValue ClearedBase = DAG.getNode(ISD::AND, dl, BaseVT, Base, ClearMask);
4325
4326 SDValue ExtVal = DAG.getZExtOrTrunc(Val, dl, BaseVT);
4327 SDValue ShiftedVal =
4328 DAG.getNode(ISD::SHL, dl, BaseVT, ExtVal, LegalShiftAmount);
4329 SDValue Result = DAG.getNode(ISD::OR, dl, BaseVT, ClearedBase, ShiftedVal);
4330 setValue(&I, Result);
4331}
4332
4333void SelectionDAGBuilder::visitBitExtract(const User &I) {
4334 SDValue Src = getValue(I.getOperand(0));
4335 SDValue Offset = getValue(I.getOperand(1));
4336 EVT SrcVT = Src.getValueType();
4337 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4338 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4339 SDLoc dl = getCurSDLoc();
4340
4341 assert(ResultVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() &&
4342 "bitextract result wider than source should be rejected by verifier");
4343
4344 // Legalize shift amount to the target's shift amount type.
4345 EVT ShiftAmtTy = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
4346 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Offset, dl, ShiftAmtTy);
4347
4348 // Shift right by Offset - brings target field to bit 0
4349 SDValue Shifted = DAG.getNode(ISD::SRL, dl, SrcVT, Src, LegalShiftAmount);
4350
4352 if (!ResultVT.isInteger()) {
4353 // Drop into the integer domain to safely truncate the shifted bits
4354 EVT IntResultVT = ResultVT.changeTypeToInteger();
4355 Result = DAG.getNode(ISD::TRUNCATE, dl, IntResultVT, Shifted);
4356 Result = DAG.getBitcast(ResultVT, Result);
4357 } else {
4358 // Normal integer path
4359 Result = DAG.getNode(ISD::TRUNCATE, dl, ResultVT, Shifted);
4360 }
4361
4362 setValue(&I, Result);
4363}
4364
4365void SelectionDAGBuilder::visitExtractElement(const User &I) {
4366 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4367 SDValue InVec = getValue(I.getOperand(0));
4368 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
4369 TLI.getVectorIdxTy(DAG.getDataLayout()));
4371 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4372 InVec, InIdx));
4373}
4374
4375void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4376 SDValue Src1 = getValue(I.getOperand(0));
4377 SDValue Src2 = getValue(I.getOperand(1));
4378 ArrayRef<int> Mask;
4379 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
4380 Mask = SVI->getShuffleMask();
4381 else
4382 Mask = cast<ConstantExpr>(I).getShuffleMask();
4383 SDLoc DL = getCurSDLoc();
4384 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4385 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4386 EVT SrcVT = Src1.getValueType();
4387
4388 if (all_of(Mask, equal_to(0)) && VT.isScalableVector()) {
4389 // Canonical splat form of first element of first input vector.
4390 SDValue FirstElt =
4391 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
4392 DAG.getVectorIdxConstant(0, DL));
4393 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4394 return;
4395 }
4396
4397 // For now, we only handle splats for scalable vectors.
4398 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4399 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4400 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4401
4402 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4403 unsigned MaskNumElts = Mask.size();
4404
4405 if (SrcNumElts == MaskNumElts) {
4406 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4407 return;
4408 }
4409
4410 // Normalize the shuffle vector since mask and vector length don't match.
4411 if (SrcNumElts < MaskNumElts) {
4412 // Mask is longer than the source vectors. We can use concatenate vector to
4413 // make the mask and vectors lengths match.
4414
4415 if (MaskNumElts % SrcNumElts == 0) {
4416 // Mask length is a multiple of the source vector length.
4417 // Check if the shuffle is some kind of concatenation of the input
4418 // vectors.
4419 unsigned NumConcat = MaskNumElts / SrcNumElts;
4420 bool IsConcat = true;
4421 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4422 for (unsigned i = 0; i != MaskNumElts; ++i) {
4423 int Idx = Mask[i];
4424 if (Idx < 0)
4425 continue;
4426 // Ensure the indices in each SrcVT sized piece are sequential and that
4427 // the same source is used for the whole piece.
4428 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4429 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4430 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4431 IsConcat = false;
4432 break;
4433 }
4434 // Remember which source this index came from.
4435 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4436 }
4437
4438 // The shuffle is concatenating multiple vectors together. Just emit
4439 // a CONCAT_VECTORS operation.
4440 if (IsConcat) {
4441 SmallVector<SDValue, 8> ConcatOps;
4442 for (auto Src : ConcatSrcs) {
4443 if (Src < 0)
4444 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4445 else if (Src == 0)
4446 ConcatOps.push_back(Src1);
4447 else
4448 ConcatOps.push_back(Src2);
4449 }
4450 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4451 return;
4452 }
4453 }
4454
4455 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4456 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4457 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4458 PaddedMaskNumElts);
4459
4460 // Pad both vectors with undefs to make them the same length as the mask.
4461 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4462
4463 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4464 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4465 MOps1[0] = Src1;
4466 MOps2[0] = Src2;
4467
4468 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4469 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4470
4471 // Readjust mask for new input vector length.
4472 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4473 for (unsigned i = 0; i != MaskNumElts; ++i) {
4474 int Idx = Mask[i];
4475 if (Idx >= (int)SrcNumElts)
4476 Idx -= SrcNumElts - PaddedMaskNumElts;
4477 MappedOps[i] = Idx;
4478 }
4479
4480 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4481
4482 // If the concatenated vector was padded, extract a subvector with the
4483 // correct number of elements.
4484 if (MaskNumElts != PaddedMaskNumElts)
4485 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
4486 DAG.getVectorIdxConstant(0, DL));
4487
4488 setValue(&I, Result);
4489 return;
4490 }
4491
4492 assert(SrcNumElts > MaskNumElts);
4493
4494 // Analyze the access pattern of the vector to see if we can extract
4495 // two subvectors and do the shuffle.
4496 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4497 bool CanExtract = true;
4498 for (int Idx : Mask) {
4499 unsigned Input = 0;
4500 if (Idx < 0)
4501 continue;
4502
4503 if (Idx >= (int)SrcNumElts) {
4504 Input = 1;
4505 Idx -= SrcNumElts;
4506 }
4507
4508 // If all the indices come from the same MaskNumElts sized portion of
4509 // the sources we can use extract. Also make sure the extract wouldn't
4510 // extract past the end of the source.
4511 int NewStartIdx = alignDown(Idx, MaskNumElts);
4512 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4513 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4514 CanExtract = false;
4515 // Make sure we always update StartIdx as we use it to track if all
4516 // elements are undef.
4517 StartIdx[Input] = NewStartIdx;
4518 }
4519
4520 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4521 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4522 return;
4523 }
4524 if (CanExtract) {
4525 // Extract appropriate subvector and generate a vector shuffle
4526 for (unsigned Input = 0; Input < 2; ++Input) {
4527 SDValue &Src = Input == 0 ? Src1 : Src2;
4528 if (StartIdx[Input] < 0)
4529 Src = DAG.getUNDEF(VT);
4530 else {
4531 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4532 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4533 }
4534 }
4535
4536 // Calculate new mask.
4537 SmallVector<int, 8> MappedOps(Mask);
4538 for (int &Idx : MappedOps) {
4539 if (Idx >= (int)SrcNumElts)
4540 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4541 else if (Idx >= 0)
4542 Idx -= StartIdx[0];
4543 }
4544
4545 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4546 return;
4547 }
4548
4549 // We can't use either concat vectors or extract subvectors so fall back to
4550 // replacing the shuffle with extract and build vector.
4551 // to insert and build vector.
4552 EVT EltVT = VT.getVectorElementType();
4554 for (int Idx : Mask) {
4555 SDValue Res;
4556
4557 if (Idx < 0) {
4558 Res = DAG.getUNDEF(EltVT);
4559 } else {
4560 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4561 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4562
4563 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4564 DAG.getVectorIdxConstant(Idx, DL));
4565 }
4566
4567 Ops.push_back(Res);
4568 }
4569
4570 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4571}
4572
4573void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4574 ArrayRef<unsigned> Indices = I.getIndices();
4575 const Value *Op0 = I.getOperand(0);
4576 const Value *Op1 = I.getOperand(1);
4577 Type *AggTy = I.getType();
4578 Type *ValTy = Op1->getType();
4579 bool IntoUndef = isa<UndefValue>(Op0);
4580 bool FromUndef = isa<UndefValue>(Op1);
4581
4582 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4583
4584 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4585 SmallVector<EVT, 4> AggValueVTs;
4586 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4587 SmallVector<EVT, 4> ValValueVTs;
4588 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4589
4590 unsigned NumAggValues = AggValueVTs.size();
4591 unsigned NumValValues = ValValueVTs.size();
4592 SmallVector<SDValue, 4> Values(NumAggValues);
4593
4594 // Ignore an insertvalue that produces an empty object
4595 if (!NumAggValues) {
4596 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4597 return;
4598 }
4599
4600 SDValue Agg = getValue(Op0);
4601 unsigned i = 0;
4602 // Copy the beginning value(s) from the original aggregate.
4603 for (; i != LinearIndex; ++i)
4604 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4605 SDValue(Agg.getNode(), Agg.getResNo() + i);
4606 // Copy values from the inserted value(s).
4607 if (NumValValues) {
4608 SDValue Val = getValue(Op1);
4609 for (; i != LinearIndex + NumValValues; ++i)
4610 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4611 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4612 }
4613 // Copy remaining value(s) from the original aggregate.
4614 for (; i != NumAggValues; ++i)
4615 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4616 SDValue(Agg.getNode(), Agg.getResNo() + i);
4617
4619 DAG.getVTList(AggValueVTs), Values));
4620}
4621
4622void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4623 ArrayRef<unsigned> Indices = I.getIndices();
4624 const Value *Op0 = I.getOperand(0);
4625 Type *AggTy = Op0->getType();
4626 Type *ValTy = I.getType();
4627 bool OutOfUndef = isa<UndefValue>(Op0);
4628
4629 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4630
4631 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4632 SmallVector<EVT, 4> ValValueVTs;
4633 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4634
4635 unsigned NumValValues = ValValueVTs.size();
4636
4637 // Ignore a extractvalue that produces an empty object
4638 if (!NumValValues) {
4639 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4640 return;
4641 }
4642
4643 SmallVector<SDValue, 4> Values(NumValValues);
4644
4645 SDValue Agg = getValue(Op0);
4646 // Copy out the selected value(s).
4647 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4648 Values[i - LinearIndex] =
4649 OutOfUndef ?
4650 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4651 SDValue(Agg.getNode(), Agg.getResNo() + i);
4652
4654 DAG.getVTList(ValValueVTs), Values));
4655}
4656
4657void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4658 Value *Op0 = I.getOperand(0);
4659 // Note that the pointer operand may be a vector of pointers. Take the scalar
4660 // element which holds a pointer.
4661 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4662 SDValue N = getValue(Op0);
4663 SDLoc dl = getCurSDLoc();
4664 auto &TLI = DAG.getTargetLoweringInfo();
4665 GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();
4666
4667 // For a vector GEP, keep the prefix scalar as long as possible, then
4668 // convert any scalars encountered after the first vector operand to vectors.
4669 bool IsVectorGEP = I.getType()->isVectorTy();
4670 ElementCount VectorElementCount =
4671 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4673
4675 GTI != E; ++GTI) {
4676 const Value *Idx = GTI.getOperand();
4677 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4678 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4679 if (Field) {
4680 // N = N + Offset
4682 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
4683
4684 // In an inbounds GEP with an offset that is nonnegative even when
4685 // interpreted as signed, assume there is no unsigned overflow.
4686 SDNodeFlags Flags;
4687 if (NW.hasNoUnsignedWrap() ||
4688 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4690 Flags.setInBounds(NW.isInBounds());
4691
4692 N = DAG.getMemBasePlusOffset(
4693 N, DAG.getConstant(Offset, dl, N.getValueType()), dl, Flags);
4694 }
4695 } else {
4696 // IdxSize is the width of the arithmetic according to IR semantics.
4697 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4698 // (and fix up the result later).
4699 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4700 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4701 TypeSize ElementSize =
4702 GTI.getSequentialElementStride(DAG.getDataLayout());
4703 // We intentionally mask away the high bits here; ElementSize may not
4704 // fit in IdxTy.
4705 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4706 /*isSigned=*/false, /*implicitTrunc=*/true);
4707 bool ElementScalable = ElementSize.isScalable();
4708
4709 // If this is a scalar constant or a splat vector of constants,
4710 // handle it quickly.
4711 const auto *C = dyn_cast<Constant>(Idx);
4712 if (C && isa<VectorType>(C->getType()))
4713 C = C->getSplatValue();
4714
4715 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4716 if (CI && CI->isZero())
4717 continue;
4718 if (CI && !ElementScalable) {
4719 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4720 LLVMContext &Context = *DAG.getContext();
4721 SDValue OffsVal;
4722 if (N.getValueType().isVector())
4723 OffsVal = DAG.getConstant(
4724 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4725 else
4726 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4727
4728 // In an inbounds GEP with an offset that is nonnegative even when
4729 // interpreted as signed, assume there is no unsigned overflow.
4730 SDNodeFlags Flags;
4731 if (NW.hasNoUnsignedWrap() ||
4732 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4733 Flags.setNoUnsignedWrap(true);
4734 Flags.setInBounds(NW.isInBounds());
4735
4736 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4737
4738 N = DAG.getMemBasePlusOffset(N, OffsVal, dl, Flags);
4739 continue;
4740 }
4741
4742 // N = N + Idx * ElementMul;
4743 SDValue IdxN = getValue(Idx);
4744
4745 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
4746 if (N.getValueType().isVector()) {
4747 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
4748 VectorElementCount);
4749 IdxN = DAG.getSplat(VT, dl, IdxN);
4750 } else {
4751 EVT VT =
4752 EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
4753 N = DAG.getSplat(VT, dl, N);
4754 }
4755 }
4756
4757 // If the index is smaller or larger than intptr_t, truncate or extend
4758 // it.
4759 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4760
4761 SDNodeFlags ScaleFlags;
4762 // The multiplication of an index by the type size does not wrap the
4763 // pointer index type in a signed sense (mul nsw).
4765
4766 // The multiplication of an index by the type size does not wrap the
4767 // pointer index type in an unsigned sense (mul nuw).
4768 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4769
4770 if (ElementScalable) {
4771 EVT VScaleTy = N.getValueType().getScalarType();
4772 SDValue VScale = DAG.getNode(
4773 ISD::VSCALE, dl, VScaleTy,
4774 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4775 if (N.getValueType().isVector())
4776 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4777 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale,
4778 ScaleFlags);
4779 } else {
4780 // If this is a multiply by a power of two, turn it into a shl
4781 // immediately. This is a very common case.
4782 if (ElementMul != 1) {
4783 if (ElementMul.isPowerOf2()) {
4784 unsigned Amt = ElementMul.logBase2();
4785 IdxN = DAG.getNode(
4786 ISD::SHL, dl, N.getValueType(), IdxN,
4787 DAG.getShiftAmountConstant(Amt, N.getValueType(), dl),
4788 ScaleFlags);
4789 } else {
4790 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4791 IdxN.getValueType());
4792 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, Scale,
4793 ScaleFlags);
4794 }
4795 }
4796 }
4797
4798 // The successive addition of the current address, truncated to the
4799 // pointer index type and interpreted as an unsigned number, and each
4800 // offset, also interpreted as an unsigned number, does not wrap the
4801 // pointer index type (add nuw).
4802 SDNodeFlags AddFlags;
4803 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4804 AddFlags.setInBounds(NW.isInBounds());
4805
4806 N = DAG.getMemBasePlusOffset(N, IdxN, dl, AddFlags);
4807 }
4808 }
4809
4810 if (IsVectorGEP && !N.getValueType().isVector()) {
4811 EVT VT = EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
4812 N = DAG.getSplat(VT, dl, N);
4813 }
4814
4815 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4816 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4817 if (IsVectorGEP) {
4818 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4819 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4820 }
4821
4822 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4823 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4824
4825 setValue(&I, N);
4826}
4827
4828void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4829 // If this is a fixed sized alloca in the entry block of the function,
4830 // allocate it statically on the stack.
4831 if (FuncInfo.StaticAllocaMap.count(&I))
4832 return; // getValue will auto-populate this.
4833
4834 SDLoc dl = getCurSDLoc();
4835 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4836 auto &DL = DAG.getDataLayout();
4837 TypeSize TySize = I.getAllocationBaseSize(DL);
4838 MaybeAlign Alignment = I.getAlign();
4839
4840 SDValue AllocSize = getValue(I.getArraySize());
4841
4842 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4843 if (AllocSize.getValueType() != IntPtr)
4844 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4845
4846 AllocSize = DAG.getNode(
4847 ISD::MUL, dl, IntPtr, AllocSize,
4848 DAG.getZExtOrTrunc(DAG.getTypeSize(dl, MVT::i64, TySize), dl, IntPtr));
4849
4850 // Handle alignment. If the requested alignment is less than or equal to
4851 // the stack alignment, ignore it. If the size is greater than or equal to
4852 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4853 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4854 if (*Alignment <= StackAlign)
4855 Alignment = std::nullopt;
4856
4857 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4858 // Round the size of the allocation up to the stack alignment size
4859 // by add SA-1 to the size. This doesn't overflow because we're computing
4860 // an address inside an alloca.
4861 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4862 DAG.getConstant(StackAlignMask, dl, IntPtr),
4864
4865 // Mask out the low bits for alignment purposes.
4866 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4867 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
4868
4869 SDValue Ops[] = {
4870 getRoot(), AllocSize,
4871 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4872 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4873 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4874 setValue(&I, DSA);
4875 DAG.setRoot(DSA.getValue(1));
4876
4877 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4878}
4879
4880static const MDNode *getRangeMetadata(const Instruction &I) {
4881 return I.getMetadata(LLVMContext::MD_range);
4882}
4883
4884static std::optional<ConstantRange> getRange(const Instruction &I) {
4885 if (const auto *CB = dyn_cast<CallBase>(&I))
4886 if (std::optional<ConstantRange> CR = CB->getRange())
4887 return CR;
4888 if (const MDNode *Range = getRangeMetadata(I))
4890 return std::nullopt;
4891}
4892
4894 if (const auto *CB = dyn_cast<CallBase>(&I))
4895 return CB->getRetNoFPClass();
4896 return fcNone;
4897}
4898
4899void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4900 if (I.isAtomic())
4901 return visitAtomicLoad(I);
4902
4903 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4904 const Value *SV = I.getOperand(0);
4905 if (TLI.supportSwiftError()) {
4906 // Swifterror values can come from either a function parameter with
4907 // swifterror attribute or an alloca with swifterror attribute.
4908 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4909 if (Arg->hasSwiftErrorAttr())
4910 return visitLoadFromSwiftError(I);
4911 }
4912
4913 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4914 if (Alloca->isSwiftError())
4915 return visitLoadFromSwiftError(I);
4916 }
4917 }
4918
4919 SDValue Ptr = getValue(SV);
4920
4921 Type *Ty = I.getType();
4922 SmallVector<EVT, 4> ValueVTs, MemVTs;
4924 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4925 unsigned NumValues = ValueVTs.size();
4926 if (NumValues == 0)
4927 return;
4928
4929 Align Alignment = I.getAlign();
4930 AAMDNodes AAInfo = I.getAAMetadata();
4931 const MDNode *Ranges = getRangeMetadata(I);
4932 const MDNode *MemCacheHint = getMemCacheHint(I);
4933 bool isVolatile = I.isVolatile();
4934 MachineMemOperand::Flags MMOFlags =
4935 TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
4936
4937 SDValue Root;
4938 bool ConstantMemory = false;
4939 if (isVolatile)
4940 // Serialize volatile loads with other side effects.
4941 Root = getRoot();
4942 else if (NumValues > MaxParallelChains)
4943 Root = getMemoryRoot();
4944 else if (BatchAA &&
4945 BatchAA->pointsToConstantMemory(MemoryLocation(
4946 SV,
4947 LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4948 AAInfo))) {
4949 // Do not serialize (non-volatile) loads of constant memory with anything.
4950 Root = DAG.getEntryNode();
4951 ConstantMemory = true;
4953 } else {
4954 // Do not serialize non-volatile loads against each other.
4955 Root = DAG.getRoot();
4956 }
4957
4958 SDLoc dl = getCurSDLoc();
4959
4960 if (isVolatile)
4961 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4962
4964 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4965
4966 unsigned ChainI = 0;
4967 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4968 // Serializing loads here may result in excessive register pressure, and
4969 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4970 // could recover a bit by hoisting nodes upward in the chain by recognizing
4971 // they are side-effect free or do not alias. The optimizer should really
4972 // avoid this case by converting large object/array copies to llvm.memcpy
4973 // (MaxParallelChains should always remain as failsafe).
4974 if (ChainI == MaxParallelChains) {
4975 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4976 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4977 ArrayRef(Chains.data(), ChainI));
4978 Root = Chain;
4979 ChainI = 0;
4980 }
4981
4982 // TODO: MachinePointerInfo only supports a fixed length offset.
4983 MachinePointerInfo PtrInfo =
4984 !Offsets[i].isScalable() || Offsets[i].isZero()
4985 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4986 : MachinePointerInfo();
4987
4988 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4989 SDValue L =
4990 DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment, MMOFlags,
4991 MMOMetadata(AAInfo, Ranges, MemCacheHint));
4992 Chains[ChainI] = L.getValue(1);
4993
4994 if (MemVTs[i] != ValueVTs[i])
4995 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4996
4997 if (MDNode *NoFPClassMD = I.getMetadata(LLVMContext::MD_nofpclass)) {
4998 uint64_t FPTestInt =
4999 cast<ConstantInt>(
5000 cast<ConstantAsMetadata>(NoFPClassMD->getOperand(0))->getValue())
5001 ->getZExtValue();
5002 if (FPTestInt != fcNone) {
5003 SDValue FPTestConst =
5004 DAG.getTargetConstant(FPTestInt, SDLoc(), MVT::i32);
5005 L = DAG.getNode(ISD::AssertNoFPClass, dl, L.getValueType(), L,
5006 FPTestConst);
5007 }
5008 }
5009 Values[i] = L;
5010 }
5011
5012 if (!ConstantMemory) {
5013 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5014 ArrayRef(Chains.data(), ChainI));
5015 if (isVolatile)
5016 DAG.setRoot(Chain);
5017 else
5018 PendingLoads.push_back(Chain);
5019 }
5020
5021 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
5022 DAG.getVTList(ValueVTs), Values));
5023}
5024
5025void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
5026 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5027 "call visitStoreToSwiftError when backend supports swifterror");
5028
5029 SmallVector<EVT, 4> ValueVTs;
5030 SmallVector<uint64_t, 4> Offsets;
5031 const Value *SrcV = I.getOperand(0);
5032 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
5033 SrcV->getType(), ValueVTs, /*MemVTs=*/nullptr, &Offsets, 0);
5034 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5035 "expect a single EVT for swifterror");
5036
5037 SDValue Src = getValue(SrcV);
5038 // Create a virtual register, then update the virtual register.
5039 Register VReg =
5040 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
5041 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
5042 // Chain can be getRoot or getControlRoot.
5043 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
5044 SDValue(Src.getNode(), Src.getResNo()));
5045 DAG.setRoot(CopyNode);
5046}
5047
5048void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
5049 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5050 "call visitLoadFromSwiftError when backend supports swifterror");
5051
5052 assert(!I.isVolatile() &&
5053 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
5054 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
5055 "Support volatile, non temporal, invariant for load_from_swift_error");
5056
5057 const Value *SV = I.getOperand(0);
5058 Type *Ty = I.getType();
5059 assert(
5060 (!BatchAA ||
5061 !BatchAA->pointsToConstantMemory(MemoryLocation(
5062 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
5063 I.getAAMetadata()))) &&
5064 "load_from_swift_error should not be constant memory");
5065
5066 SmallVector<EVT, 4> ValueVTs;
5067 SmallVector<uint64_t, 4> Offsets;
5068 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
5069 ValueVTs, /*MemVTs=*/nullptr, &Offsets, 0);
5070 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5071 "expect a single EVT for swifterror");
5072
5073 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
5074 SDValue L = DAG.getCopyFromReg(
5075 getRoot(), getCurSDLoc(),
5076 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
5077
5078 setValue(&I, L);
5079}
5080
5081void SelectionDAGBuilder::visitStore(const StoreInst &I) {
5082 if (I.isAtomic())
5083 return visitAtomicStore(I);
5084
5085 const Value *SrcV = I.getOperand(0);
5086 const Value *PtrV = I.getOperand(1);
5087
5088 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5089 if (TLI.supportSwiftError()) {
5090 // Swifterror values can come from either a function parameter with
5091 // swifterror attribute or an alloca with swifterror attribute.
5092 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
5093 if (Arg->hasSwiftErrorAttr())
5094 return visitStoreToSwiftError(I);
5095 }
5096
5097 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
5098 if (Alloca->isSwiftError())
5099 return visitStoreToSwiftError(I);
5100 }
5101 }
5102
5103 SmallVector<EVT, 4> ValueVTs, MemVTs;
5105 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
5106 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
5107 unsigned NumValues = ValueVTs.size();
5108 if (NumValues == 0)
5109 return;
5110
5111 // Get the lowered operands. Note that we do this after
5112 // checking if NumResults is zero, because with zero results
5113 // the operands won't have values in the map.
5114 SDValue Src = getValue(SrcV);
5115 SDValue Ptr = getValue(PtrV);
5116
5117 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
5118 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
5119 SDLoc dl = getCurSDLoc();
5120 Align Alignment = I.getAlign();
5121 AAMDNodes AAInfo = I.getAAMetadata();
5122 const MDNode *MemCacheHint = getMemCacheHint(I, I.getPointerOperandIndex());
5123
5124 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
5125
5126 unsigned ChainI = 0;
5127 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
5128 // See visitLoad comments.
5129 if (ChainI == MaxParallelChains) {
5130 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5131 ArrayRef(Chains.data(), ChainI));
5132 Root = Chain;
5133 ChainI = 0;
5134 }
5135
5136 // TODO: MachinePointerInfo only supports a fixed length offset.
5137 MachinePointerInfo PtrInfo =
5138 !Offsets[i].isScalable() || Offsets[i].isZero()
5139 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
5140 : MachinePointerInfo();
5141
5142 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
5143 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
5144 if (MemVTs[i] != ValueVTs[i])
5145 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
5146 SDValue St =
5147 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags,
5148 MMOMetadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
5149 Chains[ChainI] = St;
5150 }
5151
5152 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5153 ArrayRef(Chains.data(), ChainI));
5154 setValue(&I, StoreNode);
5155 DAG.setRoot(StoreNode);
5156}
5157
5158void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
5159 bool IsCompressing) {
5160 SDLoc sdl = getCurSDLoc();
5161
5162 Value *Src0Operand = I.getArgOperand(0);
5163 Value *PtrOperand = I.getArgOperand(1);
5164 Value *MaskOperand = I.getArgOperand(2);
5165 Align Alignment = I.getParamAlign(1).valueOrOne();
5166
5167 SDValue Ptr = getValue(PtrOperand);
5168 SDValue Src0 = getValue(Src0Operand);
5169 SDValue Mask = getValue(MaskOperand);
5170 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
5171
5172 EVT VT = Src0.getValueType();
5173
5174 const auto &TLI = DAG.getTargetLoweringInfo();
5175
5176 auto MMOFlags = MachineMemOperand::MOStore;
5177 MMOFlags |= TLI.getTargetMMOFlags(I);
5178 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5180
5181 const MDNode *MemCacheHint = getMemCacheHint(I, /*OperandNo=*/1);
5182
5183 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5184 MachinePointerInfo(PtrOperand), MMOFlags,
5185 LocationSize::upperBound(VT.getStoreSize()), Alignment,
5186 MMOMetadata(I.getAAMetadata(), /*Ranges=*/nullptr, MemCacheHint));
5187
5188 SDValue StoreNode =
5189 !IsCompressing && TTI->hasConditionalLoadStoreForType(
5190 I.getArgOperand(0)->getType(), /*IsStore=*/true)
5191 ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0,
5192 Mask)
5193 : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask,
5194 VT, MMO, ISD::UNINDEXED, /*Truncating=*/false,
5195 IsCompressing);
5196 DAG.setRoot(StoreNode);
5197 setValue(&I, StoreNode);
5198}
5199
5200// Get a uniform base for the Gather/Scatter intrinsic.
5201// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
5202// We try to represent it as a base pointer + vector of indices.
5203// Usually, the vector of pointers comes from a 'getelementptr' instruction.
5204// The first operand of the GEP may be a single pointer or a vector of pointers
5205// Example:
5206// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
5207// or
5208// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
5209// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
5210//
5211// When the first GEP operand is a single pointer - it is the uniform base we
5212// are looking for. If first operand of the GEP is a splat vector - we
5213// extract the splat value and use it as a uniform base.
5214// In all other cases the function returns 'false'.
5215static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
5216 SDValue &Scale, SelectionDAGBuilder *SDB,
5217 const BasicBlock *CurBB, uint64_t ElemSize) {
5218 SelectionDAG& DAG = SDB->DAG;
5220 const DataLayout &DL = DAG.getDataLayout();
5221
5222 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
5223
5224 // Handle splat constant pointer.
5225 if (auto *C = dyn_cast<Constant>(Ptr)) {
5226 C = C->getSplatValue();
5227 if (!C)
5228 return false;
5229
5230 Base = SDB->getValue(C);
5231
5232 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
5233 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
5234 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
5235 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
5236 return true;
5237 }
5238
5240 if (!GEP || GEP->getParent() != CurBB)
5241 return false;
5242
5243 if (GEP->getNumOperands() != 2)
5244 return false;
5245
5246 const Value *BasePtr = GEP->getPointerOperand();
5247 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
5248
5249 // Make sure the base is scalar and the index is a vector.
5250 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
5251 return false;
5252
5253 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
5254 if (ScaleVal.isScalable())
5255 return false;
5256
5257 // Target may not support the required addressing mode.
5258 if (ScaleVal != 1 &&
5259 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
5260 return false;
5261
5262 Base = SDB->getValue(BasePtr);
5263 Index = SDB->getValue(IndexVal);
5264
5265 Scale =
5266 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
5267 return true;
5268}
5269
5270void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
5271 SDLoc sdl = getCurSDLoc();
5272
5273 // llvm.masked.scatter.*(Src0, Ptrs, Mask)
5274 const Value *Ptr = I.getArgOperand(1);
5275 SDValue Src0 = getValue(I.getArgOperand(0));
5276 SDValue Mask = getValue(I.getArgOperand(2));
5277 EVT VT = Src0.getValueType();
5278 Align Alignment = I.getParamAlign(1).valueOrOne();
5279 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5280
5281 SDValue Base;
5282 SDValue Index;
5283 SDValue Scale;
5284 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
5285 I.getParent(), VT.getScalarStoreSize());
5286
5287 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5288 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5289 MachinePointerInfo(AS), MachineMemOperand::MOStore,
5290 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
5291 if (!UniformBase) {
5292 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5293 Index = getValue(Ptr);
5294 Scale =
5295 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5296 }
5297
5298 EVT IdxVT = Index.getValueType();
5299 EVT EltTy = IdxVT.getVectorElementType();
5300 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5301 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
5302 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5303 }
5304
5305 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
5306 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
5307 Ops, MMO, ISD::SIGNED_SCALED, false);
5308 DAG.setRoot(Scatter);
5309 setValue(&I, Scatter);
5310}
5311
5312void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
5313 SDLoc sdl = getCurSDLoc();
5314
5315 Value *PtrOperand = I.getArgOperand(0);
5316 Value *MaskOperand = I.getArgOperand(1);
5317 Value *Src0Operand = I.getArgOperand(2);
5318 Align Alignment = I.getParamAlign(0).valueOrOne();
5319
5320 SDValue Ptr = getValue(PtrOperand);
5321 SDValue Src0 = getValue(Src0Operand);
5322 SDValue Mask = getValue(MaskOperand);
5323 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
5324
5325 EVT VT = Src0.getValueType();
5326 AAMDNodes AAInfo = I.getAAMetadata();
5327 const MDNode *Ranges = getRangeMetadata(I);
5328 const MDNode *MemCacheHint = getMemCacheHint(I, /*OperandNo=*/0);
5329
5330 // Do not serialize masked loads of constant memory with anything.
5331 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
5332 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
5333
5334 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5335
5336 const auto &TLI = DAG.getTargetLoweringInfo();
5337
5338 auto MMOFlags = MachineMemOperand::MOLoad;
5339 MMOFlags |= TLI.getTargetMMOFlags(I);
5340 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5342 if (I.hasMetadata(LLVMContext::MD_invariant_load))
5344
5345 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5346 MachinePointerInfo(PtrOperand), MMOFlags,
5347 LocationSize::upperBound(VT.getStoreSize()), Alignment,
5348 MMOMetadata(AAInfo, Ranges, MemCacheHint));
5349
5350 // The Load/Res may point to different values and both of them are output
5351 // variables.
5352 SDValue Load;
5353 SDValue Res;
5354 if (!IsExpanding &&
5355 TTI->hasConditionalLoadStoreForType(Src0Operand->getType(),
5356 /*IsStore=*/false))
5357 Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask);
5358 else
5359 Res = Load =
5360 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
5361 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5362 if (AddToChain)
5363 PendingLoads.push_back(Load.getValue(1));
5364 setValue(&I, Res);
5365}
5366
5367void SelectionDAGBuilder::visitSpeculativeLoad(const CallInst &I) {
5368 SDLoc sdl = getCurSDLoc();
5369 Value *PtrOperand = I.getArgOperand(0);
5370 // The remaining arguments (num_accessible_bytes or oracle function + args)
5371 // are IR-level semantics only; they are not needed at codegen.
5372 SDValue Ptr = getValue(PtrOperand);
5373
5374 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5375 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5376 Align Alignment = I.getParamAlign(0).valueOrOne();
5377 AAMDNodes AAInfo = I.getAAMetadata();
5378
5379 SDValue InChain = DAG.getRoot();
5380
5381 // Use MOLoad but NOT MODereferenceable - the memory may not be
5382 // fully dereferenceable.
5383 auto MMOFlags = MachineMemOperand::MOLoad;
5384 MMOFlags |= TLI.getTargetMMOFlags(I);
5385 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5387 if (I.hasMetadata(LLVMContext::MD_invariant_load))
5389
5390 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5391 MachinePointerInfo(PtrOperand), MMOFlags,
5392 LocationSize::precise(VT.getStoreSize()), Alignment, AAInfo);
5393
5394 SDValue Load = DAG.getLoad(VT, sdl, InChain, Ptr, MMO);
5395 PendingLoads.push_back(Load.getValue(1));
5396 setValue(&I, Load);
5397}
5398
5399void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5400 SDLoc sdl = getCurSDLoc();
5401
5402 // @llvm.masked.gather.*(Ptrs, Mask, Src0)
5403 const Value *Ptr = I.getArgOperand(0);
5404 SDValue Src0 = getValue(I.getArgOperand(2));
5405 SDValue Mask = getValue(I.getArgOperand(1));
5406
5407 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5408 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5409 Align Alignment = I.getParamAlign(0).valueOrOne();
5410
5411 const MDNode *Ranges = getRangeMetadata(I);
5412
5413 SDValue Root = DAG.getRoot();
5414 SDValue Base;
5415 SDValue Index;
5416 SDValue Scale;
5417 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
5418 I.getParent(), VT.getScalarStoreSize());
5419 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5420 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5421 MachinePointerInfo(AS), MachineMemOperand::MOLoad,
5423 MMOMetadata(I.getAAMetadata(), Ranges));
5424
5425 if (!UniformBase) {
5426 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5427 Index = getValue(Ptr);
5428 Scale =
5429 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5430 }
5431
5432 EVT IdxVT = Index.getValueType();
5433 EVT EltTy = IdxVT.getVectorElementType();
5434 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5435 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
5436 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5437 }
5438
5439 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5440 SDValue Gather =
5441 DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, Ops, MMO,
5443
5444 PendingLoads.push_back(Gather.getValue(1));
5445 setValue(&I, Gather);
5446}
5447
5448void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5449 SDLoc dl = getCurSDLoc();
5450 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5451 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5452 SyncScope::ID SSID = I.getSyncScopeID();
5453
5454 SDValue InChain = getRoot();
5455
5456 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
5457 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5458
5459 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5460 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
5461
5462 MachineFunction &MF = DAG.getMachineFunction();
5463 const MDNode *MemCacheHint = getMemCacheHint(I);
5464 MachineMemOperand *MMO = MF.getMachineMemOperand(
5465 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5466 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5467 SSID, SuccessOrdering, FailureOrdering);
5468
5470 dl, MemVT, VTs, InChain,
5471 getValue(I.getPointerOperand()),
5472 getValue(I.getCompareOperand()),
5473 getValue(I.getNewValOperand()), MMO);
5474
5475 SDValue OutChain = L.getValue(2);
5476
5477 setValue(&I, L);
5478 DAG.setRoot(OutChain);
5479}
5480
5481void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5482 SDLoc dl = getCurSDLoc();
5483 ISD::NodeType NT = getAtomicRMWOpcode(I.getOperation());
5484 AtomicOrdering Ordering = I.getOrdering();
5485 SyncScope::ID SSID = I.getSyncScopeID();
5486
5487 SDValue InChain = getRoot();
5488
5489 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5490 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5491 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
5492
5493 MachineFunction &MF = DAG.getMachineFunction();
5494 const MDNode *MemCacheHint = getMemCacheHint(I);
5495 MachineMemOperand *MMO = MF.getMachineMemOperand(
5496 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5497 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5498 SSID, Ordering);
5499
5500 SDValue L =
5501 DAG.getAtomic(NT, dl, MemVT, InChain,
5502 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5503 MMO);
5504
5505 SDValue OutChain = L.getValue(1);
5506
5507 setValue(&I, L);
5508 DAG.setRoot(OutChain);
5509}
5510
5511void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5512 SDLoc dl = getCurSDLoc();
5513 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5514 SDValue Ops[3];
5515 Ops[0] = getRoot();
5516 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5517 TLI.getFenceOperandTy(DAG.getDataLayout()));
5518 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5519 TLI.getFenceOperandTy(DAG.getDataLayout()));
5520 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5521 setValue(&I, N);
5522 DAG.setRoot(N);
5523}
5524
5525void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5526 SDLoc dl = getCurSDLoc();
5527 AtomicOrdering Order = I.getOrdering();
5528 SyncScope::ID SSID = I.getSyncScopeID();
5529
5530 SDValue InChain = getRoot();
5531
5532 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5533 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5534 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5535
5536 if (!TLI.isAtomicAlignmentSupported(I.getAlign(), MemVT.getSizeInBits() / 8))
5537 report_fatal_error("Cannot generate unaligned atomic load");
5538
5539 auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
5540
5541 const MDNode *Ranges = getRangeMetadata(I);
5542 const MDNode *MemCacheHint = getMemCacheHint(I);
5543 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5544 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5545 I.getAlign(), MMOMetadata(AAMDNodes(), Ranges, MemCacheHint), SSID,
5546 Order);
5547
5548 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5549
5550 SDValue Ptr = getValue(I.getPointerOperand());
5551 SDValue L =
5552 DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, MemVT, MemVT, InChain, Ptr, MMO);
5553
5554 SDValue OutChain = L.getValue(1);
5555 if (MemVT != VT)
5556 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5557
5558 setValue(&I, L);
5559 DAG.setRoot(OutChain);
5560}
5561
5562void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5563 SDLoc dl = getCurSDLoc();
5564
5565 AtomicOrdering Ordering = I.getOrdering();
5566 SyncScope::ID SSID = I.getSyncScopeID();
5567
5568 SDValue InChain = getRoot();
5569
5570 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5571 EVT MemVT =
5572 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5573
5574 if (!TLI.isAtomicAlignmentSupported(I.getAlign(), MemVT.getSizeInBits() / 8))
5575 report_fatal_error("Cannot generate unaligned atomic store");
5576
5577 auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
5578
5579 MachineFunction &MF = DAG.getMachineFunction();
5580 const MDNode *MemCacheHint = getMemCacheHint(I, I.getPointerOperandIndex());
5581 MachineMemOperand *MMO = MF.getMachineMemOperand(
5582 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5583 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5584 SSID, Ordering);
5585
5586 SDValue Val = getValue(I.getValueOperand());
5587 if (Val.getValueType() != MemVT)
5588 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5589 SDValue Ptr = getValue(I.getPointerOperand());
5590
5591 SDValue OutChain =
5592 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5593
5594 setValue(&I, OutChain);
5595 DAG.setRoot(OutChain);
5596}
5597
5598/// Check if this intrinsic call depends on the chain (1st return value)
5599/// and if it only *loads* memory.
5600/// Ignore the callsite's attributes. A specific call site may be marked with
5601/// readnone, but the lowering code will expect the chain based on the
5602/// definition.
5603std::pair<bool, bool>
5604SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {
5605 const Function *F = I.getCalledFunction();
5606 bool HasChain = !F->doesNotAccessMemory();
5607 bool OnlyLoad =
5608 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5609
5610 return {HasChain, OnlyLoad};
5611}
5612
5613SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
5614 const CallBase &I, bool HasChain, bool OnlyLoad,
5615 TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
5616 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5617
5618 // Build the operand list.
5620 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5621 if (OnlyLoad) {
5622 // We don't need to serialize loads against other loads.
5623 Ops.push_back(DAG.getRoot());
5624 } else {
5625 Ops.push_back(getRoot());
5626 }
5627 }
5628
5629 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5630 if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||
5631 TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)
5632 Ops.push_back(DAG.getTargetConstant(I.getIntrinsicID(), getCurSDLoc(),
5633 TLI.getPointerTy(DAG.getDataLayout())));
5634
5635 // Add all operands of the call to the operand list.
5636 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5637 const Value *Arg = I.getArgOperand(i);
5638 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5639 Ops.push_back(getValue(Arg));
5640 continue;
5641 }
5642
5643 // Use TargetConstant instead of a regular constant for immarg.
5644 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5645 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5646 assert(CI->getBitWidth() <= 64 &&
5647 "large intrinsic immediates not handled");
5648 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5649 } else {
5650 Ops.push_back(
5651 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5652 }
5653 }
5654
5655 if (std::optional<OperandBundleUse> Bundle =
5656 I.getOperandBundle(LLVMContext::OB_deactivation_symbol)) {
5657 auto *Sym = Bundle->Inputs[0].get();
5658 SDValue SDSym = getValue(Sym);
5659 SDSym = DAG.getDeactivationSymbol(cast<GlobalValue>(Sym));
5660 Ops.push_back(SDSym);
5661 }
5662
5663 if (std::optional<OperandBundleUse> Bundle =
5664 I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5665 Value *Token = Bundle->Inputs[0].get();
5666 SDValue ConvControlToken = getValue(Token);
5667 assert(Ops.back().getValueType() != MVT::Glue &&
5668 "Did not expect another glue node here.");
5669 ConvControlToken =
5670 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5671 Ops.push_back(ConvControlToken);
5672 }
5673
5674 return Ops;
5675}
5676
5677SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
5678 bool HasChain) {
5679 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5680
5681 SmallVector<EVT, 4> ValueVTs;
5682 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5683
5684 if (HasChain)
5685 ValueVTs.push_back(MVT::Other);
5686
5687 return DAG.getVTList(ValueVTs);
5688}
5689
5690/// Get an INTRINSIC node for a target intrinsic which does not touch memory.
5691SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5692 const Type &IntrinsicVT, bool HasChain, ArrayRef<SDValue> Ops,
5693 const SDVTList &VTs) {
5694 if (!HasChain)
5695 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
5696 if (!IntrinsicVT.isVoidTy())
5697 return DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
5698 return DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
5699}
5700
5701/// Set root, convert return type if necessary and check alignment.
5702SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
5703 bool HasChain,
5704 bool OnlyLoad,
5705 SDValue Result) {
5706 if (HasChain) {
5707 SDValue Chain = Result.getValue(Result.getNode()->getNumValues() - 1);
5708 if (OnlyLoad)
5709 PendingLoads.push_back(Chain);
5710 else
5711 DAG.setRoot(Chain);
5712 }
5713
5714 if (I.getType()->isVoidTy())
5715 return Result;
5716
5717 if (MaybeAlign Alignment = I.getRetAlign(); InsertAssertAlign && Alignment) {
5718 // Insert `assertalign` node if there's an alignment.
5719 Result = DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5720 } else if (!isa<VectorType>(I.getType())) {
5721 Result = lowerRangeToAssertZExt(DAG, I, Result);
5722 }
5723
5724 return Result;
5725}
5726
5727/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5728/// node.
5729void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5730 unsigned Intrinsic) {
5731 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
5732 Intrinsic::ID IntrinsicID = static_cast<Intrinsic::ID>(Intrinsic);
5733
5734 if (!DAG.getMachineFunction().getSubtarget().isIntrinsicSupported(
5735 Intrinsic)) {
5736 SDLoc DL = getCurSDLoc();
5737 DAG.getContext()->diagnose(DiagnosticInfoUnsupportedTargetIntrinsic(
5738 *I.getFunction(), IntrinsicID, DL.getDebugLoc()));
5739
5740 // The intrinsic is not available on this subtarget. Preserve the chain for
5741 // side-effecting intrinsics and lower any result to poison so that
5742 // compilation can continue and collect further diagnostics.
5743 if (HasChain && !OnlyLoad)
5744 DAG.setRoot(getRoot());
5745
5747 return;
5748 }
5749
5750 // Infos is set by getTgtMemIntrinsic.
5752 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5753 TLI.getTgtMemIntrinsic(Infos, I, DAG.getMachineFunction(), Intrinsic);
5754 // Use the first (primary) info determines the node opcode.
5755 TargetLowering::IntrinsicInfo *Info = !Infos.empty() ? &Infos[0] : nullptr;
5756
5758 getTargetIntrinsicOperands(I, HasChain, OnlyLoad, Info);
5759 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
5760
5761 // Propagate fast-math-flags from IR to node(s).
5762 SDNodeFlags Flags;
5763 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5764 Flags.copyFMF(*FPMO);
5765 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5766
5767 // Create the node.
5769
5770 // In some cases, custom collection of operands from CallInst I may be needed.
5772 if (!Infos.empty()) {
5773 // This is target intrinsic that touches memory
5774 // Create MachineMemOperands for each memory access described by the target.
5775 MachineFunction &MF = DAG.getMachineFunction();
5777 for (const auto &Info : Infos) {
5778 // TODO: We currently just fallback to address space 0 if
5779 // getTgtMemIntrinsic didn't yield anything useful.
5780 MachinePointerInfo MPI;
5781 if (Info.ptrVal)
5782 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5783 else if (Info.fallbackAddressSpace)
5784 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5785 EVT MemVT = Info.memVT;
5786 LocationSize Size = LocationSize::precise(Info.size);
5787 if (Size.hasValue() && !Size.getValue())
5789 Align Alignment = Info.align.value_or(DAG.getEVTAlign(MemVT));
5790 MachineMemOperand *MMO = MF.getMachineMemOperand(
5791 MPI, Info.flags, Size, Alignment, I.getAAMetadata(), Info.ssid,
5792 Info.order, Info.failureOrder);
5793 MMOs.push_back(MMO);
5794 }
5795
5796 Result = DAG.getMemIntrinsicNode(Info->opc, getCurSDLoc(), VTs, Ops,
5797 Info->memVT, MMOs);
5798 } else {
5799 Result = getTargetNonMemIntrinsicNode(*I.getType(), HasChain, Ops, VTs);
5800 }
5801
5802 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
5803
5804 setValue(&I, Result);
5805}
5806
5807/// GetSignificand - Get the significand and build it into a floating-point
5808/// number with exponent of 1:
5809///
5810/// Op = (Op & 0x007fffff) | 0x3f800000;
5811///
5812/// where Op is the hexadecimal representation of floating point value.
5814 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5815 DAG.getConstant(0x007fffff, dl, MVT::i32));
5816 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5817 DAG.getConstant(0x3f800000, dl, MVT::i32));
5818 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5819}
5820
5821/// GetExponent - Get the exponent:
5822///
5823/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5824///
5825/// where Op is the hexadecimal representation of floating point value.
5827 const TargetLowering &TLI, const SDLoc &dl) {
5828 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5829 DAG.getConstant(0x7f800000, dl, MVT::i32));
5830 SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
5831 DAG.getShiftAmountConstant(23, MVT::i32, dl));
5832 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5833 DAG.getConstant(127, dl, MVT::i32));
5834 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5835}
5836
5837/// getF32Constant - Get 32-bit floating point constant.
5838static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
5839 const SDLoc &dl) {
5840 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5841 MVT::f32);
5842}
5843
5845 SelectionDAG &DAG) {
5846 // TODO: What fast-math-flags should be set on the floating-point nodes?
5847
5848 // IntegerPartOfX = ((int32_t)(t0);
5849 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5850
5851 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5852 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5853 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5854
5855 // IntegerPartOfX <<= 23;
5856 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5857 DAG.getShiftAmountConstant(23, MVT::i32, dl));
5858
5859 SDValue TwoToFractionalPartOfX;
5860 if (LimitFloatPrecision <= 6) {
5861 // For floating-point precision of 6:
5862 //
5863 // TwoToFractionalPartOfX =
5864 // 0.997535578f +
5865 // (0.735607626f + 0.252464424f * x) * x;
5866 //
5867 // error 0.0144103317, which is 6 bits
5868 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5869 getF32Constant(DAG, 0x3e814304, dl));
5870 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5871 getF32Constant(DAG, 0x3f3c50c8, dl));
5872 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5873 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5874 getF32Constant(DAG, 0x3f7f5e7e, dl));
5875 } else if (LimitFloatPrecision <= 12) {
5876 // For floating-point precision of 12:
5877 //
5878 // TwoToFractionalPartOfX =
5879 // 0.999892986f +
5880 // (0.696457318f +
5881 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5882 //
5883 // error 0.000107046256, which is 13 to 14 bits
5884 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5885 getF32Constant(DAG, 0x3da235e3, dl));
5886 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5887 getF32Constant(DAG, 0x3e65b8f3, dl));
5888 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5889 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5890 getF32Constant(DAG, 0x3f324b07, dl));
5891 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5892 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5893 getF32Constant(DAG, 0x3f7ff8fd, dl));
5894 } else { // LimitFloatPrecision <= 18
5895 // For floating-point precision of 18:
5896 //
5897 // TwoToFractionalPartOfX =
5898 // 0.999999982f +
5899 // (0.693148872f +
5900 // (0.240227044f +
5901 // (0.554906021e-1f +
5902 // (0.961591928e-2f +
5903 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5904 // error 2.47208000*10^(-7), which is better than 18 bits
5905 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5906 getF32Constant(DAG, 0x3924b03e, dl));
5907 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5908 getF32Constant(DAG, 0x3ab24b87, dl));
5909 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5910 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5911 getF32Constant(DAG, 0x3c1d8c17, dl));
5912 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5913 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5914 getF32Constant(DAG, 0x3d634a1d, dl));
5915 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5916 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5917 getF32Constant(DAG, 0x3e75fe14, dl));
5918 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5919 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5920 getF32Constant(DAG, 0x3f317234, dl));
5921 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5922 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5923 getF32Constant(DAG, 0x3f800000, dl));
5924 }
5925
5926 // Add the exponent into the result in integer domain.
5927 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5928 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5929 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5930}
5931
5932/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5933/// limited-precision mode.
5935 const TargetLowering &TLI, SDNodeFlags Flags) {
5936 if (Op.getValueType() == MVT::f32 &&
5938
5939 // Put the exponent in the right bit position for later addition to the
5940 // final result:
5941 //
5942 // t0 = Op * log2(e)
5943
5944 // TODO: What fast-math-flags should be set here?
5945 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5946 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5947 return getLimitedPrecisionExp2(t0, dl, DAG);
5948 }
5949
5950 // No special expansion.
5951 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5952}
5953
5954/// expandLog - Lower a log intrinsic. Handles the special sequences for
5955/// limited-precision mode.
5957 const TargetLowering &TLI, SDNodeFlags Flags) {
5958 // TODO: What fast-math-flags should be set on the floating-point nodes?
5959
5960 if (Op.getValueType() == MVT::f32 &&
5962 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5963
5964 // Scale the exponent by log(2).
5965 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5966 SDValue LogOfExponent =
5967 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5968 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5969
5970 // Get the significand and build it into a floating-point number with
5971 // exponent of 1.
5972 SDValue X = GetSignificand(DAG, Op1, dl);
5973
5974 SDValue LogOfMantissa;
5975 if (LimitFloatPrecision <= 6) {
5976 // For floating-point precision of 6:
5977 //
5978 // LogofMantissa =
5979 // -1.1609546f +
5980 // (1.4034025f - 0.23903021f * x) * x;
5981 //
5982 // error 0.0034276066, which is better than 8 bits
5983 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5984 getF32Constant(DAG, 0xbe74c456, dl));
5985 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5986 getF32Constant(DAG, 0x3fb3a2b1, dl));
5987 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5988 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5989 getF32Constant(DAG, 0x3f949a29, dl));
5990 } else if (LimitFloatPrecision <= 12) {
5991 // For floating-point precision of 12:
5992 //
5993 // LogOfMantissa =
5994 // -1.7417939f +
5995 // (2.8212026f +
5996 // (-1.4699568f +
5997 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5998 //
5999 // error 0.000061011436, which is 14 bits
6000 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6001 getF32Constant(DAG, 0xbd67b6d6, dl));
6002 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6003 getF32Constant(DAG, 0x3ee4f4b8, dl));
6004 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6005 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6006 getF32Constant(DAG, 0x3fbc278b, dl));
6007 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6008 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6009 getF32Constant(DAG, 0x40348e95, dl));
6010 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6011 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6012 getF32Constant(DAG, 0x3fdef31a, dl));
6013 } else { // LimitFloatPrecision <= 18
6014 // For floating-point precision of 18:
6015 //
6016 // LogOfMantissa =
6017 // -2.1072184f +
6018 // (4.2372794f +
6019 // (-3.7029485f +
6020 // (2.2781945f +
6021 // (-0.87823314f +
6022 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
6023 //
6024 // error 0.0000023660568, which is better than 18 bits
6025 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6026 getF32Constant(DAG, 0xbc91e5ac, dl));
6027 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6028 getF32Constant(DAG, 0x3e4350aa, dl));
6029 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6030 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6031 getF32Constant(DAG, 0x3f60d3e3, dl));
6032 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6033 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6034 getF32Constant(DAG, 0x4011cdf0, dl));
6035 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6036 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6037 getF32Constant(DAG, 0x406cfd1c, dl));
6038 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6039 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
6040 getF32Constant(DAG, 0x408797cb, dl));
6041 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
6042 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
6043 getF32Constant(DAG, 0x4006dcab, dl));
6044 }
6045
6046 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
6047 }
6048
6049 // No special expansion.
6050 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
6051}
6052
6053/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
6054/// limited-precision mode.
6056 const TargetLowering &TLI, SDNodeFlags Flags) {
6057 // TODO: What fast-math-flags should be set on the floating-point nodes?
6058
6059 if (Op.getValueType() == MVT::f32 &&
6061 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
6062
6063 // Get the exponent.
6064 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
6065
6066 // Get the significand and build it into a floating-point number with
6067 // exponent of 1.
6068 SDValue X = GetSignificand(DAG, Op1, dl);
6069
6070 // Different possible minimax approximations of significand in
6071 // floating-point for various degrees of accuracy over [1,2].
6072 SDValue Log2ofMantissa;
6073 if (LimitFloatPrecision <= 6) {
6074 // For floating-point precision of 6:
6075 //
6076 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
6077 //
6078 // error 0.0049451742, which is more than 7 bits
6079 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6080 getF32Constant(DAG, 0xbeb08fe0, dl));
6081 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6082 getF32Constant(DAG, 0x40019463, dl));
6083 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6084 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6085 getF32Constant(DAG, 0x3fd6633d, dl));
6086 } else if (LimitFloatPrecision <= 12) {
6087 // For floating-point precision of 12:
6088 //
6089 // Log2ofMantissa =
6090 // -2.51285454f +
6091 // (4.07009056f +
6092 // (-2.12067489f +
6093 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
6094 //
6095 // error 0.0000876136000, which is better than 13 bits
6096 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6097 getF32Constant(DAG, 0xbda7262e, dl));
6098 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6099 getF32Constant(DAG, 0x3f25280b, dl));
6100 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6101 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6102 getF32Constant(DAG, 0x4007b923, dl));
6103 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6104 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6105 getF32Constant(DAG, 0x40823e2f, dl));
6106 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6107 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6108 getF32Constant(DAG, 0x4020d29c, dl));
6109 } else { // LimitFloatPrecision <= 18
6110 // For floating-point precision of 18:
6111 //
6112 // Log2ofMantissa =
6113 // -3.0400495f +
6114 // (6.1129976f +
6115 // (-5.3420409f +
6116 // (3.2865683f +
6117 // (-1.2669343f +
6118 // (0.27515199f -
6119 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
6120 //
6121 // error 0.0000018516, which is better than 18 bits
6122 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6123 getF32Constant(DAG, 0xbcd2769e, dl));
6124 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6125 getF32Constant(DAG, 0x3e8ce0b9, dl));
6126 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6127 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6128 getF32Constant(DAG, 0x3fa22ae7, dl));
6129 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6130 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6131 getF32Constant(DAG, 0x40525723, dl));
6132 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6133 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6134 getF32Constant(DAG, 0x40aaf200, dl));
6135 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6136 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
6137 getF32Constant(DAG, 0x40c39dad, dl));
6138 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
6139 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
6140 getF32Constant(DAG, 0x4042902c, dl));
6141 }
6142
6143 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
6144 }
6145
6146 // No special expansion.
6147 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
6148}
6149
6150/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
6151/// limited-precision mode.
6153 const TargetLowering &TLI, SDNodeFlags Flags) {
6154 // TODO: What fast-math-flags should be set on the floating-point nodes?
6155
6156 if (Op.getValueType() == MVT::f32 &&
6158 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
6159
6160 // Scale the exponent by log10(2) [0.30102999f].
6161 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
6162 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
6163 getF32Constant(DAG, 0x3e9a209a, dl));
6164
6165 // Get the significand and build it into a floating-point number with
6166 // exponent of 1.
6167 SDValue X = GetSignificand(DAG, Op1, dl);
6168
6169 SDValue Log10ofMantissa;
6170 if (LimitFloatPrecision <= 6) {
6171 // For floating-point precision of 6:
6172 //
6173 // Log10ofMantissa =
6174 // -0.50419619f +
6175 // (0.60948995f - 0.10380950f * x) * x;
6176 //
6177 // error 0.0014886165, which is 6 bits
6178 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6179 getF32Constant(DAG, 0xbdd49a13, dl));
6180 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6181 getF32Constant(DAG, 0x3f1c0789, dl));
6182 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6183 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6184 getF32Constant(DAG, 0x3f011300, dl));
6185 } else if (LimitFloatPrecision <= 12) {
6186 // For floating-point precision of 12:
6187 //
6188 // Log10ofMantissa =
6189 // -0.64831180f +
6190 // (0.91751397f +
6191 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
6192 //
6193 // error 0.00019228036, which is better than 12 bits
6194 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6195 getF32Constant(DAG, 0x3d431f31, dl));
6196 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
6197 getF32Constant(DAG, 0x3ea21fb2, dl));
6198 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6199 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
6200 getF32Constant(DAG, 0x3f6ae232, dl));
6201 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6202 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
6203 getF32Constant(DAG, 0x3f25f7c3, dl));
6204 } else { // LimitFloatPrecision <= 18
6205 // For floating-point precision of 18:
6206 //
6207 // Log10ofMantissa =
6208 // -0.84299375f +
6209 // (1.5327582f +
6210 // (-1.0688956f +
6211 // (0.49102474f +
6212 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
6213 //
6214 // error 0.0000037995730, which is better than 18 bits
6215 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6216 getF32Constant(DAG, 0x3c5d51ce, dl));
6217 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
6218 getF32Constant(DAG, 0x3e00685a, dl));
6219 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6220 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
6221 getF32Constant(DAG, 0x3efb6798, dl));
6222 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6223 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
6224 getF32Constant(DAG, 0x3f88d192, dl));
6225 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6226 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
6227 getF32Constant(DAG, 0x3fc4316c, dl));
6228 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6229 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
6230 getF32Constant(DAG, 0x3f57ce70, dl));
6231 }
6232
6233 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
6234 }
6235
6236 // No special expansion.
6237 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
6238}
6239
6240/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
6241/// limited-precision mode.
6243 const TargetLowering &TLI, SDNodeFlags Flags) {
6244 if (Op.getValueType() == MVT::f32 &&
6246 return getLimitedPrecisionExp2(Op, dl, DAG);
6247
6248 // No special expansion.
6249 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
6250}
6251
6252/// visitPow - Lower a pow intrinsic. Handles the special sequences for
6253/// limited-precision mode with x == 10.0f.
6255 SelectionDAG &DAG, const TargetLowering &TLI,
6256 SDNodeFlags Flags) {
6257 bool IsExp10 = false;
6258 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
6261 APFloat Ten(10.0f);
6262 IsExp10 = LHSC->isExactlyValue(Ten);
6263 }
6264 }
6265
6266 // TODO: What fast-math-flags should be set on the FMUL node?
6267 if (IsExp10) {
6268 // Put the exponent in the right bit position for later addition to the
6269 // final result:
6270 //
6271 // #define LOG2OF10 3.3219281f
6272 // t0 = Op * LOG2OF10;
6273 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
6274 getF32Constant(DAG, 0x40549a78, dl));
6275 return getLimitedPrecisionExp2(t0, dl, DAG);
6276 }
6277
6278 // No special expansion.
6279 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
6280}
6281
6282/// ExpandPowI - Expand a llvm.powi intrinsic.
6284 SelectionDAG &DAG) {
6285 // If RHS is a constant, we can expand this out to a multiplication tree if
6286 // it's beneficial on the target, otherwise we end up lowering to a call to
6287 // __powidf2 (for example).
6289 unsigned Val = RHSC->getSExtValue();
6290
6291 // powi(x, 0) -> 1.0
6292 if (Val == 0)
6293 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
6294
6296 Val, DAG.shouldOptForSize())) {
6297 // Get the exponent as a positive value.
6298 if ((int)Val < 0)
6299 Val = -Val;
6300 // We use the simple binary decomposition method to generate the multiply
6301 // sequence. There are more optimal ways to do this (for example,
6302 // powi(x,15) generates one more multiply than it should), but this has
6303 // the benefit of being both really simple and much better than a libcall.
6304 SDValue Res; // Logically starts equal to 1.0
6305 SDValue CurSquare = LHS;
6306 // TODO: Intrinsics should have fast-math-flags that propagate to these
6307 // nodes.
6308 while (Val) {
6309 if (Val & 1) {
6310 if (Res.getNode())
6311 Res =
6312 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
6313 else
6314 Res = CurSquare; // 1.0*CurSquare.
6315 }
6316
6317 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
6318 CurSquare, CurSquare);
6319 Val >>= 1;
6320 }
6321
6322 // If the original was negative, invert the result, producing 1/(x*x*x).
6323 if (RHSC->getSExtValue() < 0)
6324 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
6325 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
6326 return Res;
6327 }
6328 }
6329
6330 // Otherwise, expand to a libcall.
6331 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
6332}
6333
6334static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
6335 SDValue LHS, SDValue RHS, SDValue Scale,
6336 SelectionDAG &DAG, const TargetLowering &TLI) {
6337 EVT VT = LHS.getValueType();
6338 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
6339 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
6340 LLVMContext &Ctx = *DAG.getContext();
6341
6342 // If the type is legal but the operation isn't, this node might survive all
6343 // the way to operation legalization. If we end up there and we do not have
6344 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
6345 // node.
6346
6347 // Coax the legalizer into expanding the node during type legalization instead
6348 // by bumping the size by one bit. This will force it to Promote, enabling the
6349 // early expansion and avoiding the need to expand later.
6350
6351 // We don't have to do this if Scale is 0; that can always be expanded, unless
6352 // it's a saturating signed operation. Those can experience true integer
6353 // division overflow, a case which we must avoid.
6354
6355 // FIXME: We wouldn't have to do this (or any of the early
6356 // expansion/promotion) if it was possible to expand a libcall of an
6357 // illegal type during operation legalization. But it's not, so things
6358 // get a bit hacky.
6359 unsigned ScaleInt = Scale->getAsZExtVal();
6360 if ((ScaleInt > 0 || (Saturating && Signed)) &&
6361 (TLI.isTypeLegal(VT) ||
6362 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
6363 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
6364 Opcode, VT, ScaleInt);
6365 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
6366 EVT PromVT;
6367 if (VT.isScalarInteger())
6368 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
6369 else if (VT.isVector()) {
6370 PromVT = VT.getVectorElementType();
6371 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
6372 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
6373 } else
6374 llvm_unreachable("Wrong VT for DIVFIX?");
6375 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
6376 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
6377 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
6378 // For saturating operations, we need to shift up the LHS to get the
6379 // proper saturation width, and then shift down again afterwards.
6380 if (Saturating)
6381 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
6382 DAG.getConstant(1, DL, ShiftTy));
6383 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
6384 if (Saturating)
6385 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
6386 DAG.getConstant(1, DL, ShiftTy));
6387 return DAG.getZExtOrTrunc(Res, DL, VT);
6388 }
6389 }
6390
6391 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
6392}
6393
6394// getUnderlyingArgRegs - Find underlying registers used for a truncated,
6395// bitcasted, or split argument. Returns a list of <Register, size in bits>
6396static void
6397getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
6398 const SDValue &N) {
6399 switch (N.getOpcode()) {
6400 case ISD::CopyFromReg: {
6401 SDValue Op = N.getOperand(1);
6402 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
6403 Op.getValueType().getSizeInBits());
6404 return;
6405 }
6406 case ISD::BITCAST:
6407 case ISD::AssertZext:
6408 case ISD::AssertSext:
6409 case ISD::TRUNCATE:
6410 getUnderlyingArgRegs(Regs, N.getOperand(0));
6411 return;
6412 case ISD::BUILD_PAIR:
6413 case ISD::BUILD_VECTOR:
6415 for (SDValue Op : N->op_values())
6416 getUnderlyingArgRegs(Regs, Op);
6417 return;
6418 default:
6419 return;
6420 }
6421}
6422
6423/// If the DbgValueInst is a dbg_value of a function argument, create the
6424/// corresponding DBG_VALUE machine instruction for it now. At the end of
6425/// instruction selection, they will be inserted to the entry BB.
6426/// We don't currently support this for variadic dbg_values, as they shouldn't
6427/// appear for function arguments or in the prologue.
6428bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6429 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6430 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6431 const Argument *Arg = dyn_cast<Argument>(V);
6432 if (!Arg)
6433 return false;
6434
6435 MachineFunction &MF = DAG.getMachineFunction();
6436 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6437
6438 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6439 // we've been asked to pursue.
6440 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6441 bool Indirect) {
6442 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6443 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6444 // pointing at the VReg, which will be patched up later.
6445 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
6447 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6448 /* isKill */ false, /* isDead */ false,
6449 /* isUndef */ false, /* isEarlyClobber */ false,
6450 /* SubReg */ 0, /* isDebug */ true)});
6451
6452 auto *NewDIExpr = FragExpr;
6453 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6454 // the DIExpression.
6455 if (Indirect)
6456 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
6458 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
6459 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
6460 } else {
6461 // Create a completely standard DBG_VALUE.
6462 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
6463 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
6464 }
6465 };
6466
6467 if (Kind == FuncArgumentDbgValueKind::Value) {
6468 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6469 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6470 // the entry block.
6471 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6472 if (!IsInEntryBlock)
6473 return false;
6474
6475 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6476 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6477 // variable that also is a param.
6478 //
6479 // Although, if we are at the top of the entry block already, we can still
6480 // emit using ArgDbgValue. This might catch some situations when the
6481 // dbg.value refers to an argument that isn't used in the entry block, so
6482 // any CopyToReg node would be optimized out and the only way to express
6483 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6484 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6485 // we should only emit as ArgDbgValue if the Variable is an argument to the
6486 // current function, and the dbg.value intrinsic is found in the entry
6487 // block.
6488 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6489 !DL->getInlinedAt();
6490 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6491 if (!IsInPrologue && !VariableIsFunctionInputArg)
6492 return false;
6493
6494 // Here we assume that a function argument on IR level only can be used to
6495 // describe one input parameter on source level. If we for example have
6496 // source code like this
6497 //
6498 // struct A { long x, y; };
6499 // void foo(struct A a, long b) {
6500 // ...
6501 // b = a.x;
6502 // ...
6503 // }
6504 //
6505 // and IR like this
6506 //
6507 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6508 // entry:
6509 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6510 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6511 // call void @llvm.dbg.value(metadata i32 %b, "b",
6512 // ...
6513 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6514 // ...
6515 //
6516 // then the last dbg.value is describing a parameter "b" using a value that
6517 // is an argument. But since we already has used %a1 to describe a parameter
6518 // we should not handle that last dbg.value here (that would result in an
6519 // incorrect hoisting of the DBG_VALUE to the function entry).
6520 // Notice that we allow one dbg.value per IR level argument, to accommodate
6521 // for the situation with fragments above.
6522 // If there is no node for the value being handled, we return true to skip
6523 // the normal generation of debug info, as it would kill existing debug
6524 // info for the parameter in case of duplicates.
6525 if (VariableIsFunctionInputArg) {
6526 unsigned ArgNo = Arg->getArgNo();
6527 if (ArgNo >= FuncInfo.DescribedArgs.size())
6528 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6529 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6530 return !NodeMap[V].getNode();
6531 FuncInfo.DescribedArgs.set(ArgNo);
6532 }
6533 }
6534
6535 bool IsIndirect = false;
6536 std::optional<MachineOperand> Op;
6537 // Some arguments' frame index is recorded during argument lowering.
6538 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6539 if (FI != std::numeric_limits<int>::max())
6541
6543 if (!Op && N.getNode()) {
6544 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6545 Register Reg;
6546 if (ArgRegsAndSizes.size() == 1)
6547 Reg = ArgRegsAndSizes.front().first;
6548
6549 if (Reg && Reg.isVirtual()) {
6550 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6551 Register PR = RegInfo.getLiveInPhysReg(Reg);
6552 if (PR)
6553 Reg = PR;
6554 }
6555 if (Reg) {
6557 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6558 }
6559 }
6560
6561 if (!Op && N.getNode()) {
6562 // Check if frame index is available.
6563 SDValue LCandidate = peekThroughBitcasts(N);
6564 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6565 if (FrameIndexSDNode *FINode =
6566 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6567 Op = MachineOperand::CreateFI(FINode->getIndex());
6568 }
6569
6570 if (!Op) {
6571 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6572 auto splitMultiRegDbgValue =
6573 [&](ArrayRef<std::pair<Register, TypeSize>> SplitRegs) -> bool {
6574 unsigned Offset = 0;
6575 for (const auto &[Reg, RegSizeInBits] : SplitRegs) {
6576 // FIXME: Scalable sizes are not supported in fragment expressions.
6577 if (RegSizeInBits.isScalable())
6578 return false;
6579
6580 // If the expression is already a fragment, the current register
6581 // offset+size might extend beyond the fragment. In this case, only
6582 // the register bits that are inside the fragment are relevant.
6583 int RegFragmentSizeInBits = RegSizeInBits.getFixedValue();
6584 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6585 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6586 // The register is entirely outside the expression fragment,
6587 // so is irrelevant for debug info.
6588 if (Offset >= ExprFragmentSizeInBits)
6589 break;
6590 // The register is partially outside the expression fragment, only
6591 // the low bits within the fragment are relevant for debug info.
6592 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6593 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6594 }
6595 }
6596
6597 auto FragmentExpr = DIExpression::createFragmentExpression(
6598 Expr, Offset, RegFragmentSizeInBits);
6599 Offset += RegSizeInBits.getFixedValue();
6600 // If a valid fragment expression cannot be created, the variable's
6601 // correct value cannot be determined and so it is set as poison.
6602 if (!FragmentExpr) {
6603 SDDbgValue *SDV = DAG.getConstantDbgValue(
6604 Variable, Expr, PoisonValue::get(V->getType()), DL, SDNodeOrder);
6605 DAG.AddDbgValue(SDV, false);
6606 continue;
6607 }
6608 MachineInstr *NewMI = MakeVRegDbgValue(
6609 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6610 FuncInfo.ArgDbgValues.push_back(NewMI);
6611 }
6612
6613 return true;
6614 };
6615
6616 // Check if ValueMap has reg number.
6618 VMI = FuncInfo.ValueMap.find(V);
6619 if (VMI != FuncInfo.ValueMap.end()) {
6620 const auto &TLI = DAG.getTargetLoweringInfo();
6621 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6622 V->getType(), std::nullopt);
6623 if (RFV.occupiesMultipleRegs())
6624 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6625
6626 Op = MachineOperand::CreateReg(VMI->second, false);
6627 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6628 } else if (ArgRegsAndSizes.size() > 1) {
6629 // This was split due to the calling convention, and no virtual register
6630 // mapping exists for the value.
6631 return splitMultiRegDbgValue(ArgRegsAndSizes);
6632 }
6633 }
6634
6635 if (!Op)
6636 return false;
6637
6638 assert(Variable->isValidLocationForIntrinsic(DL) &&
6639 "Expected inlined-at fields to agree");
6640 MachineInstr *NewMI = nullptr;
6641
6642 if (Op->isReg())
6643 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6644 else
6645 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6646 Variable, Expr);
6647
6648 // Otherwise, use ArgDbgValues.
6649 FuncInfo.ArgDbgValues.push_back(NewMI);
6650 return true;
6651}
6652
6653/// Return the appropriate SDDbgValue based on N.
6654SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6655 DILocalVariable *Variable,
6656 DIExpression *Expr,
6657 const DebugLoc &dl,
6658 unsigned DbgSDNodeOrder) {
6659 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6660 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6661 // stack slot locations.
6662 //
6663 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6664 // debug values here after optimization:
6665 //
6666 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6667 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6668 //
6669 // Both describe the direct values of their associated variables.
6670 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6671 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6672 }
6673 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6674 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6675}
6676
6677static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6678 switch (Intrinsic) {
6679 case Intrinsic::smul_fix:
6680 return ISD::SMULFIX;
6681 case Intrinsic::umul_fix:
6682 return ISD::UMULFIX;
6683 case Intrinsic::smul_fix_sat:
6684 return ISD::SMULFIXSAT;
6685 case Intrinsic::umul_fix_sat:
6686 return ISD::UMULFIXSAT;
6687 case Intrinsic::sdiv_fix:
6688 return ISD::SDIVFIX;
6689 case Intrinsic::udiv_fix:
6690 return ISD::UDIVFIX;
6691 case Intrinsic::sdiv_fix_sat:
6692 return ISD::SDIVFIXSAT;
6693 case Intrinsic::udiv_fix_sat:
6694 return ISD::UDIVFIXSAT;
6695 default:
6696 llvm_unreachable("Unhandled fixed point intrinsic");
6697 }
6698}
6699
6700/// Given a @llvm.call.preallocated.setup, return the corresponding
6701/// preallocated call.
6702static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6703 assert(cast<CallBase>(PreallocatedSetup)
6705 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6706 "expected call_preallocated_setup Value");
6707 for (const auto *U : PreallocatedSetup->users()) {
6708 auto *UseCall = cast<CallBase>(U);
6709 const Function *Fn = UseCall->getCalledFunction();
6710 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6711 return UseCall;
6712 }
6713 }
6714 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6715}
6716
6717/// If DI is a debug value with an EntryValue expression, lower it using the
6718/// corresponding physical register of the associated Argument value
6719/// (guaranteed to exist by the verifier).
6720bool SelectionDAGBuilder::visitEntryValueDbgValue(
6722 DIExpression *Expr, DebugLoc DbgLoc) {
6723 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6724 return false;
6725
6726 // These properties are guaranteed by the verifier.
6727 const Argument *Arg = cast<Argument>(Values[0]);
6728 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6729
6730 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6731 if (ArgIt == FuncInfo.ValueMap.end()) {
6732 LLVM_DEBUG(
6733 dbgs() << "Dropping dbg.value: expression is entry_value but "
6734 "couldn't find an associated register for the Argument\n");
6735 return true;
6736 }
6737 Register ArgVReg = ArgIt->getSecond();
6738
6739 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6740 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6741 SDDbgValue *SDV = DAG.getVRegDbgValue(
6742 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6743 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6744 return true;
6745 }
6746 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6747 "couldn't find a physical register\n");
6748 return true;
6749}
6750
6751/// Lower the call to the specified intrinsic function.
6752void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6753 unsigned Intrinsic) {
6754 SDLoc sdl = getCurSDLoc();
6755 switch (Intrinsic) {
6756 case Intrinsic::experimental_convergence_anchor:
6757 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6758 break;
6759 case Intrinsic::experimental_convergence_entry:
6760 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6761 break;
6762 case Intrinsic::experimental_convergence_loop: {
6763 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6764 auto *Token = Bundle->Inputs[0].get();
6765 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6766 getValue(Token)));
6767 break;
6768 }
6769 }
6770}
6771
6772void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6773 unsigned IntrinsicID) {
6774 // For now, we're only lowering an 'add' histogram.
6775 // We can add others later, e.g. saturating adds, min/max.
6776 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6777 "Tried to lower unsupported histogram type");
6778 SDLoc sdl = getCurSDLoc();
6779 Value *Ptr = I.getOperand(0);
6780 SDValue Inc = getValue(I.getOperand(1));
6781 SDValue Mask = getValue(I.getOperand(2));
6782
6783 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6784 EVT VT = Inc.getValueType();
6785 Align Alignment = DAG.getEVTAlign(VT);
6786
6787 const MDNode *Ranges = getRangeMetadata(I);
6788
6789 SDValue Root = DAG.getRoot();
6790 SDValue Base;
6791 SDValue Index;
6792 SDValue Scale;
6793 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
6794 I.getParent(), VT.getScalarStoreSize());
6795
6796 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6797
6798 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6799 MachinePointerInfo(AS),
6801 MemoryLocation::UnknownSize, Alignment,
6802 MMOMetadata(I.getAAMetadata(), Ranges));
6803
6804 if (!UniformBase) {
6805 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
6806 Index = getValue(Ptr);
6807 Scale =
6808 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
6809 }
6810
6811 EVT IdxVT = Index.getValueType();
6812
6813 // Avoid using e.g. i32 as index type when the increment must be performed
6814 // on i64's.
6815 bool MustExtendIndex = VT.getScalarSizeInBits() > IdxVT.getScalarSizeInBits();
6816 EVT EltTy = MustExtendIndex ? VT : IdxVT.getVectorElementType();
6817 if (MustExtendIndex || TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
6818 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
6819 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
6820 }
6821
6822 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6823
6824 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6825 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
6826 Ops, MMO, ISD::SIGNED_SCALED);
6827
6828 setValue(&I, Histogram);
6829 DAG.setRoot(Histogram);
6830}
6831
6832void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6833 unsigned Intrinsic) {
6834 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6835 "Tried lowering invalid vector extract last");
6836 SDLoc sdl = getCurSDLoc();
6837 const DataLayout &Layout = DAG.getDataLayout();
6838 SDValue Data = getValue(I.getOperand(0));
6839 SDValue Mask = getValue(I.getOperand(1));
6840
6841 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6842 EVT ResVT = TLI.getValueType(Layout, I.getType());
6843
6844 EVT ExtVT = TLI.getVectorIdxTy(Layout);
6845 SDValue Idx = DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, sdl, ExtVT, Mask);
6846 SDValue Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl, ResVT, Data, Idx);
6847
6848 Value *Default = I.getOperand(2);
6850 SDValue PassThru = getValue(Default);
6851 EVT BoolVT = Mask.getValueType().getScalarType();
6852 SDValue AnyActive = DAG.getNode(ISD::VECREDUCE_OR, sdl, BoolVT, Mask);
6853 Result = DAG.getSelect(sdl, ResVT, AnyActive, Result, PassThru);
6854 }
6855
6856 setValue(&I, Result);
6857}
6858
6859/// Lower the call to the specified intrinsic function.
6860void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6861 unsigned Intrinsic) {
6862 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6863 SDLoc sdl = getCurSDLoc();
6864 DebugLoc dl = getCurDebugLoc();
6865 SDValue Res;
6866
6867 SDNodeFlags Flags;
6868 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6869 Flags.copyFMF(*FPOp);
6870
6871 switch (Intrinsic) {
6872 default:
6873 // By default, turn this into a target intrinsic node.
6874 visitTargetIntrinsic(I, Intrinsic);
6875 return;
6876 case Intrinsic::vscale: {
6877 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6878 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6879 return;
6880 }
6881 case Intrinsic::vastart: visitVAStart(I); return;
6882 case Intrinsic::vaend: visitVAEnd(I); return;
6883 case Intrinsic::vacopy: visitVACopy(I); return;
6884 case Intrinsic::returnaddress:
6885 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
6886 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6887 getValue(I.getArgOperand(0))));
6888 return;
6889 case Intrinsic::addressofreturnaddress:
6890 setValue(&I,
6891 DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
6892 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6893 return;
6894 case Intrinsic::sponentry:
6895 setValue(&I,
6896 DAG.getNode(ISD::SPONENTRY, sdl,
6897 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6898 return;
6899 case Intrinsic::frameaddress:
6900 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
6901 TLI.getFrameIndexTy(DAG.getDataLayout()),
6902 getValue(I.getArgOperand(0))));
6903 return;
6904 case Intrinsic::read_volatile_register:
6905 case Intrinsic::read_register: {
6906 Value *Reg = I.getArgOperand(0);
6907 SDValue Chain = getRoot();
6909 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6910 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6911 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6912 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6913 setValue(&I, Res);
6914 DAG.setRoot(Res.getValue(1));
6915 return;
6916 }
6917 case Intrinsic::write_register: {
6918 Value *Reg = I.getArgOperand(0);
6919 Value *RegValue = I.getArgOperand(1);
6920 SDValue Chain = getRoot();
6922 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6923 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6924 RegName, getValue(RegValue)));
6925 return;
6926 }
6927 case Intrinsic::write_volatile_register: {
6928 Value *Reg = I.getArgOperand(0);
6929 Value *RegValue = I.getArgOperand(1);
6930 SDValue Chain = getRoot();
6931 const MDNode *MD = cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata());
6932 SDValue RegName = DAG.getMDNode(MD);
6933 EVT VT = TLI.getValueType(DAG.getDataLayout(), RegValue->getType());
6934 SDValue WriteChain = DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other,
6935 Chain, RegName, getValue(RegValue));
6936 // FAKE_USE of the physical register marks it live after the WRITE_REGISTER,
6937 // preventing the backend from dead-eliminating the write. This is
6938 // preferred over READ_REGISTER, which would emit extra register copies
6939 // (e.g. fmov xN, dN for FP/SIMD registers).
6940 const MDString *RegStr = cast<MDString>(MD->getOperand(0));
6941 LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
6942 const MachineFunction &MF = DAG.getMachineFunction();
6943 Register PhysReg =
6944 TLI.getRegisterByName(RegStr->getString().data(), Ty, MF);
6945 if (PhysReg.isValid()) {
6946 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6947 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(PhysReg);
6948 MVT RegVT = *TRI->legalclasstypes_begin(*RC);
6949 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other,
6950 {WriteChain, DAG.getRegister(PhysReg, RegVT)}));
6951 } else {
6952 DAG.setRoot(WriteChain);
6953 }
6954 return;
6955 }
6956 case Intrinsic::memcpy:
6957 case Intrinsic::memcpy_inline: {
6958 const auto &MCI = cast<MemCpyInst>(I);
6959 SDValue Dst = getValue(I.getArgOperand(0));
6960 SDValue Src = getValue(I.getArgOperand(1));
6961 SDValue Size = getValue(I.getArgOperand(2));
6962 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6963 "memcpy_inline needs constant size");
6964 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6965 Align DstAlign = MCI.getDestAlign().valueOrOne();
6966 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6967 bool isVol = MCI.isVolatile();
6968 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6969 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, DstAlign, SrcAlign,
6970 isVol, MCI.isForceInlined(), &I, std::nullopt,
6971 MachinePointerInfo(I.getArgOperand(0)),
6972 MachinePointerInfo(I.getArgOperand(1)),
6973 I.getAAMetadata(), BatchAA);
6974 updateDAGForMaybeTailCall(MC);
6975 return;
6976 }
6977 case Intrinsic::memset:
6978 case Intrinsic::memset_inline: {
6979 const auto &MSII = cast<MemSetInst>(I);
6980 SDValue Dst = getValue(I.getArgOperand(0));
6981 SDValue Value = getValue(I.getArgOperand(1));
6982 SDValue Size = getValue(I.getArgOperand(2));
6983 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6984 "memset_inline needs constant size");
6985 // @llvm.memset defines 0 and 1 to both mean no alignment.
6986 Align DstAlign = MSII.getDestAlign().valueOrOne();
6987 bool isVol = MSII.isVolatile();
6988 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6989 SDValue MC = DAG.getMemset(
6990 Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(),
6991 &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6992 updateDAGForMaybeTailCall(MC);
6993 return;
6994 }
6995 case Intrinsic::memmove: {
6996 const auto &MMI = cast<MemMoveInst>(I);
6997 SDValue Op1 = getValue(I.getArgOperand(0));
6998 SDValue Op2 = getValue(I.getArgOperand(1));
6999 SDValue Op3 = getValue(I.getArgOperand(2));
7000 // @llvm.memmove defines 0 and 1 to both mean no alignment.
7001 Align DstAlign = MMI.getDestAlign().valueOrOne();
7002 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
7003 bool isVol = MMI.isVolatile();
7004 SDValue Root = isVol ? getRoot() : getMemoryRoot();
7005 SDValue MM = DAG.getMemmove(
7006 Root, sdl, Op1, Op2, Op3, DstAlign, SrcAlign, isVol, &I,
7007 /* OverrideTailCall */ std::nullopt,
7008 MachinePointerInfo(I.getArgOperand(0)),
7009 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), BatchAA);
7010 updateDAGForMaybeTailCall(MM);
7011 return;
7012 }
7013 case Intrinsic::memcpy_element_unordered_atomic: {
7014 auto &MI = cast<AnyMemCpyInst>(I);
7015 SDValue Dst = getValue(MI.getRawDest());
7016 SDValue Src = getValue(MI.getRawSource());
7017 SDValue Length = getValue(MI.getLength());
7018
7019 Type *LengthTy = MI.getLength()->getType();
7020 unsigned ElemSz = MI.getElementSizeInBytes();
7021 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7022 SDValue MC =
7023 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
7024 isTC, MachinePointerInfo(MI.getRawDest()),
7025 MachinePointerInfo(MI.getRawSource()));
7026 updateDAGForMaybeTailCall(MC);
7027 return;
7028 }
7029 case Intrinsic::memmove_element_unordered_atomic: {
7030 auto &MI = cast<AnyMemMoveInst>(I);
7031 SDValue Dst = getValue(MI.getRawDest());
7032 SDValue Src = getValue(MI.getRawSource());
7033 SDValue Length = getValue(MI.getLength());
7034
7035 Type *LengthTy = MI.getLength()->getType();
7036 unsigned ElemSz = MI.getElementSizeInBytes();
7037 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7038 SDValue MC =
7039 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
7040 isTC, MachinePointerInfo(MI.getRawDest()),
7041 MachinePointerInfo(MI.getRawSource()));
7042 updateDAGForMaybeTailCall(MC);
7043 return;
7044 }
7045 case Intrinsic::memset_element_unordered_atomic: {
7046 auto &MI = cast<AnyMemSetInst>(I);
7047 SDValue Dst = getValue(MI.getRawDest());
7048 SDValue Val = getValue(MI.getValue());
7049 SDValue Length = getValue(MI.getLength());
7050
7051 Type *LengthTy = MI.getLength()->getType();
7052 unsigned ElemSz = MI.getElementSizeInBytes();
7053 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7054 SDValue MC =
7055 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
7056 isTC, MachinePointerInfo(MI.getRawDest()));
7057 updateDAGForMaybeTailCall(MC);
7058 return;
7059 }
7060 case Intrinsic::call_preallocated_setup: {
7061 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
7062 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
7063 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
7064 getRoot(), SrcValue);
7065 setValue(&I, Res);
7066 DAG.setRoot(Res);
7067 return;
7068 }
7069 case Intrinsic::call_preallocated_arg: {
7070 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
7071 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
7072 SDValue Ops[3];
7073 Ops[0] = getRoot();
7074 Ops[1] = SrcValue;
7075 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7076 MVT::i32); // arg index
7077 SDValue Res = DAG.getNode(
7079 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
7080 setValue(&I, Res);
7081 DAG.setRoot(Res.getValue(1));
7082 return;
7083 }
7084
7085 case Intrinsic::eh_typeid_for: {
7086 // Find the type id for the given typeinfo.
7087 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
7088 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
7089 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
7090 setValue(&I, Res);
7091 return;
7092 }
7093
7094 case Intrinsic::eh_return_i32:
7095 case Intrinsic::eh_return_i64:
7096 DAG.getMachineFunction().setCallsEHReturn(true);
7097 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
7098 MVT::Other,
7100 getValue(I.getArgOperand(0)),
7101 getValue(I.getArgOperand(1))));
7102 return;
7103 case Intrinsic::eh_unwind_init:
7104 DAG.getMachineFunction().setCallsUnwindInit(true);
7105 return;
7106 case Intrinsic::eh_dwarf_cfa:
7107 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
7108 TLI.getPointerTy(DAG.getDataLayout()),
7109 getValue(I.getArgOperand(0))));
7110 return;
7111 case Intrinsic::eh_sjlj_callsite: {
7112 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
7113 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
7114
7115 FuncInfo.setCurrentCallSite(CI->getZExtValue());
7116 return;
7117 }
7118 case Intrinsic::eh_sjlj_functioncontext: {
7119 // Get and store the index of the function context.
7120 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7121 AllocaInst *FnCtx =
7122 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
7123 int FI = FuncInfo.StaticAllocaMap[FnCtx];
7125 return;
7126 }
7127 case Intrinsic::eh_sjlj_setjmp: {
7128 SDValue Ops[2];
7129 Ops[0] = getRoot();
7130 Ops[1] = getValue(I.getArgOperand(0));
7131 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
7132 DAG.getVTList(MVT::i32, MVT::Other), Ops);
7133 setValue(&I, Op.getValue(0));
7134 DAG.setRoot(Op.getValue(1));
7135 return;
7136 }
7137 case Intrinsic::eh_sjlj_longjmp:
7138 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
7139 getRoot(), getValue(I.getArgOperand(0))));
7140 return;
7141 case Intrinsic::eh_sjlj_setup_dispatch:
7142 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
7143 getRoot()));
7144 return;
7145 case Intrinsic::masked_gather:
7146 visitMaskedGather(I);
7147 return;
7148 case Intrinsic::masked_load:
7149 visitMaskedLoad(I);
7150 return;
7151 case Intrinsic::masked_scatter:
7152 visitMaskedScatter(I);
7153 return;
7154 case Intrinsic::masked_store:
7155 visitMaskedStore(I);
7156 return;
7157 case Intrinsic::masked_expandload:
7158 visitMaskedLoad(I, true /* IsExpanding */);
7159 return;
7160 case Intrinsic::masked_compressstore:
7161 visitMaskedStore(I, true /* IsCompressing */);
7162 return;
7163 case Intrinsic::speculative_load:
7164 visitSpeculativeLoad(I);
7165 return;
7166 case Intrinsic::powi:
7167 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
7168 getValue(I.getArgOperand(1)), DAG));
7169 return;
7170 case Intrinsic::log:
7171 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7172 return;
7173 case Intrinsic::log2:
7174 setValue(&I,
7175 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7176 return;
7177 case Intrinsic::log10:
7178 setValue(&I,
7179 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7180 return;
7181 case Intrinsic::exp:
7182 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7183 return;
7184 case Intrinsic::exp2:
7185 setValue(&I,
7186 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7187 return;
7188 case Intrinsic::pow:
7189 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
7190 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
7191 return;
7192 case Intrinsic::sqrt:
7193 case Intrinsic::fabs:
7194 case Intrinsic::sin:
7195 case Intrinsic::cos:
7196 case Intrinsic::tan:
7197 case Intrinsic::asin:
7198 case Intrinsic::acos:
7199 case Intrinsic::atan:
7200 case Intrinsic::sinh:
7201 case Intrinsic::cosh:
7202 case Intrinsic::tanh:
7203 case Intrinsic::exp10:
7204 case Intrinsic::floor:
7205 case Intrinsic::ceil:
7206 case Intrinsic::trunc:
7207 case Intrinsic::rint:
7208 case Intrinsic::nearbyint:
7209 case Intrinsic::round:
7210 case Intrinsic::roundeven:
7211 case Intrinsic::canonicalize: {
7212 unsigned Opcode;
7213 // clang-format off
7214 switch (Intrinsic) {
7215 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7216 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
7217 case Intrinsic::fabs: Opcode = ISD::FABS; break;
7218 case Intrinsic::sin: Opcode = ISD::FSIN; break;
7219 case Intrinsic::cos: Opcode = ISD::FCOS; break;
7220 case Intrinsic::tan: Opcode = ISD::FTAN; break;
7221 case Intrinsic::asin: Opcode = ISD::FASIN; break;
7222 case Intrinsic::acos: Opcode = ISD::FACOS; break;
7223 case Intrinsic::atan: Opcode = ISD::FATAN; break;
7224 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
7225 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
7226 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
7227 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
7228 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
7229 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
7230 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
7231 case Intrinsic::rint: Opcode = ISD::FRINT; break;
7232 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
7233 case Intrinsic::round: Opcode = ISD::FROUND; break;
7234 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
7235 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
7236 }
7237 // clang-format on
7238
7239 setValue(&I, DAG.getNode(Opcode, sdl,
7240 getValue(I.getArgOperand(0)).getValueType(),
7241 getValue(I.getArgOperand(0)), Flags));
7242 return;
7243 }
7244 case Intrinsic::atan2:
7245 setValue(&I, DAG.getNode(ISD::FATAN2, sdl,
7246 getValue(I.getArgOperand(0)).getValueType(),
7247 getValue(I.getArgOperand(0)),
7248 getValue(I.getArgOperand(1)), Flags));
7249 return;
7250 case Intrinsic::lround:
7251 case Intrinsic::llround:
7252 case Intrinsic::lrint:
7253 case Intrinsic::llrint: {
7254 unsigned Opcode;
7255 // clang-format off
7256 switch (Intrinsic) {
7257 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7258 case Intrinsic::lround: Opcode = ISD::LROUND; break;
7259 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
7260 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
7261 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
7262 }
7263 // clang-format on
7264
7265 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7266 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
7267 getValue(I.getArgOperand(0))));
7268 return;
7269 }
7270 case Intrinsic::minnum:
7271 setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
7272 getValue(I.getArgOperand(0)).getValueType(),
7273 getValue(I.getArgOperand(0)),
7274 getValue(I.getArgOperand(1)), Flags));
7275 return;
7276 case Intrinsic::maxnum:
7277 setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
7278 getValue(I.getArgOperand(0)).getValueType(),
7279 getValue(I.getArgOperand(0)),
7280 getValue(I.getArgOperand(1)), Flags));
7281 return;
7282 case Intrinsic::minimum:
7283 setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
7284 getValue(I.getArgOperand(0)).getValueType(),
7285 getValue(I.getArgOperand(0)),
7286 getValue(I.getArgOperand(1)), Flags));
7287 return;
7288 case Intrinsic::maximum:
7289 setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
7290 getValue(I.getArgOperand(0)).getValueType(),
7291 getValue(I.getArgOperand(0)),
7292 getValue(I.getArgOperand(1)), Flags));
7293 return;
7294 case Intrinsic::minimumnum:
7295 setValue(&I, DAG.getNode(ISD::FMINIMUMNUM, sdl,
7296 getValue(I.getArgOperand(0)).getValueType(),
7297 getValue(I.getArgOperand(0)),
7298 getValue(I.getArgOperand(1)), Flags));
7299 return;
7300 case Intrinsic::maximumnum:
7301 setValue(&I, DAG.getNode(ISD::FMAXIMUMNUM, sdl,
7302 getValue(I.getArgOperand(0)).getValueType(),
7303 getValue(I.getArgOperand(0)),
7304 getValue(I.getArgOperand(1)), Flags));
7305 return;
7306 case Intrinsic::copysign:
7307 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
7308 getValue(I.getArgOperand(0)).getValueType(),
7309 getValue(I.getArgOperand(0)),
7310 getValue(I.getArgOperand(1)), Flags));
7311 return;
7312 case Intrinsic::ldexp:
7313 setValue(&I, DAG.getNode(ISD::FLDEXP, sdl,
7314 getValue(I.getArgOperand(0)).getValueType(),
7315 getValue(I.getArgOperand(0)),
7316 getValue(I.getArgOperand(1)), Flags));
7317 return;
7318 case Intrinsic::modf:
7319 case Intrinsic::sincos:
7320 case Intrinsic::sincospi:
7321 case Intrinsic::frexp: {
7322 unsigned Opcode;
7323 switch (Intrinsic) {
7324 default:
7325 llvm_unreachable("unexpected intrinsic");
7326 case Intrinsic::sincos:
7327 Opcode = ISD::FSINCOS;
7328 break;
7329 case Intrinsic::sincospi:
7330 Opcode = ISD::FSINCOSPI;
7331 break;
7332 case Intrinsic::modf:
7333 Opcode = ISD::FMODF;
7334 break;
7335 case Intrinsic::frexp:
7336 Opcode = ISD::FFREXP;
7337 break;
7338 }
7339 SmallVector<EVT, 2> ValueVTs;
7340 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
7341 SDVTList VTs = DAG.getVTList(ValueVTs);
7342 setValue(
7343 &I, DAG.getNode(Opcode, sdl, VTs, getValue(I.getArgOperand(0)), Flags));
7344 return;
7345 }
7346 case Intrinsic::arithmetic_fence: {
7347 setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
7348 getValue(I.getArgOperand(0)).getValueType(),
7349 getValue(I.getArgOperand(0)), Flags));
7350 return;
7351 }
7352 case Intrinsic::fma:
7353 setValue(&I, DAG.getNode(
7354 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
7355 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
7356 getValue(I.getArgOperand(2)), Flags));
7357 return;
7358#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7359 case Intrinsic::INTRINSIC:
7360#include "llvm/IR/ConstrainedOps.def"
7361 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
7362 return;
7363#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7364#include "llvm/IR/VPIntrinsics.def"
7365 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
7366 return;
7367 case Intrinsic::fptrunc_round: {
7368 // Get the last argument, the metadata and convert it to an integer in the
7369 // call
7370 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7371 std::optional<RoundingMode> RoundMode =
7372 convertStrToRoundingMode(cast<MDString>(MD)->getString());
7373
7374 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7375
7376 // Propagate fast-math-flags from IR to node(s).
7377 SDNodeFlags Flags;
7378 Flags.copyFMF(*cast<FPMathOperator>(&I));
7379 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
7380
7382 Result = DAG.getNode(
7383 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
7384 DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));
7385 setValue(&I, Result);
7386
7387 return;
7388 }
7389 case Intrinsic::fmuladd: {
7390 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7391 if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
7392 setValue(&I, DAG.getNode(ISD::FMA, sdl,
7393 getValue(I.getArgOperand(0)).getValueType(),
7394 getValue(I.getArgOperand(0)),
7395 getValue(I.getArgOperand(1)),
7396 getValue(I.getArgOperand(2)), Flags));
7397 } else if (TLI.isOperationLegalOrCustom(ISD::FMULADD, VT)) {
7398 // TODO: Support splitting the vector.
7399 setValue(&I, DAG.getNode(ISD::FMULADD, sdl,
7400 getValue(I.getArgOperand(0)).getValueType(),
7401 getValue(I.getArgOperand(0)),
7402 getValue(I.getArgOperand(1)),
7403 getValue(I.getArgOperand(2)), Flags));
7404 } else {
7405 // TODO: Intrinsic calls should have fast-math-flags.
7406 SDValue Mul = DAG.getNode(
7407 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
7408 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
7409 SDValue Add = DAG.getNode(ISD::FADD, sdl,
7410 getValue(I.getArgOperand(0)).getValueType(),
7411 Mul, getValue(I.getArgOperand(2)), Flags);
7412 setValue(&I, Add);
7413 }
7414 return;
7415 }
7416 case Intrinsic::fptosi_sat: {
7417 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7418 setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
7419 getValue(I.getArgOperand(0)),
7420 DAG.getValueType(VT.getScalarType())));
7421 return;
7422 }
7423 case Intrinsic::fptoui_sat: {
7424 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7425 setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
7426 getValue(I.getArgOperand(0)),
7427 DAG.getValueType(VT.getScalarType())));
7428 return;
7429 }
7430 case Intrinsic::convert_from_arbitrary_fp: {
7431 // Extract format metadata and convert to semantics enum.
7432 EVT DstVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7433 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7434 StringRef FormatStr = cast<MDString>(MD)->getString();
7435 const fltSemantics *SrcSem =
7437 if (!SrcSem) {
7438 DAG.getContext()->emitError(
7439 "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7440 "'");
7441 setValue(&I, DAG.getPOISON(DstVT));
7442 return;
7443 }
7445
7446 SDValue IntVal = getValue(I.getArgOperand(0));
7447
7448 // Emit ISD::CONVERT_FROM_ARBITRARY_FP node.
7449 SDValue SemConst =
7450 DAG.getTargetConstant(static_cast<int>(SemEnum), sdl, MVT::i32);
7451 setValue(&I, DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, sdl, DstVT, IntVal,
7452 SemConst));
7453 return;
7454 }
7455 case Intrinsic::convert_to_arbitrary_fp: {
7456 // Extract format metadata and convert to semantics enum.
7457 EVT DstVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7458 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7459 StringRef FormatStr = cast<MDString>(MD)->getString();
7460 const fltSemantics *DstSem =
7462 if (!DstSem) {
7463 DAG.getContext()->emitError(
7464 "convert_to_arbitrary_fp: not implemented format '" + FormatStr +
7465 "'");
7466 setValue(&I, DAG.getPOISON(DstVT));
7467 return;
7468 }
7470
7471 Metadata *RoundMD =
7472 cast<MetadataAsValue>(I.getArgOperand(2))->getMetadata();
7473 StringRef RoundStr = cast<MDString>(RoundMD)->getString();
7474 std::optional<RoundingMode> RoundMode = convertStrToRoundingMode(RoundStr);
7475 assert(RoundMode && *RoundMode != RoundingMode::Dynamic &&
7476 "Dynamic rounding mode should have been rejected by the verifier");
7477
7478 uint64_t Saturate =
7479 cast<ConstantInt>(I.getArgOperand(3))->getZExtValue() ? 1 : 0;
7480
7481 SDValue FloatVal = getValue(I.getArgOperand(0));
7482
7483 SDValue SemConst =
7484 DAG.getTargetConstant(static_cast<int>(SemEnum), sdl, MVT::i32);
7485 SDValue RoundConst =
7486 DAG.getTargetConstant(static_cast<int>(*RoundMode), sdl, MVT::i32);
7487 SDValue SatConst = DAG.getTargetConstant(Saturate, sdl, MVT::i32);
7488 setValue(&I, DAG.getNode(ISD::CONVERT_TO_ARBITRARY_FP, sdl, DstVT, FloatVal,
7489 SemConst, RoundConst, SatConst));
7490 return;
7491 }
7492 case Intrinsic::set_rounding:
7493 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
7494 {getRoot(), getValue(I.getArgOperand(0))});
7495 setValue(&I, Res);
7496 DAG.setRoot(Res.getValue(0));
7497 return;
7498 case Intrinsic::is_fpclass: {
7499 const DataLayout DLayout = DAG.getDataLayout();
7500 EVT DestVT = TLI.getValueType(DLayout, I.getType());
7501 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
7502 FPClassTest Test = static_cast<FPClassTest>(
7503 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
7504 MachineFunction &MF = DAG.getMachineFunction();
7505 const Function &F = MF.getFunction();
7506 SDValue Op = getValue(I.getArgOperand(0));
7507 SDNodeFlags Flags;
7508 Flags.setNoFPExcept(
7509 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7510 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7511 // expansion can use illegal types. Making expansion early allows
7512 // legalizing these types prior to selection.
7513 if (!TLI.isOperationLegal(ISD::IS_FPCLASS, ArgVT) &&
7515 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
7516 setValue(&I, Result);
7517 return;
7518 }
7519
7520 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
7521 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
7522 setValue(&I, V);
7523 return;
7524 }
7525 case Intrinsic::get_fpenv: {
7526 const DataLayout DLayout = DAG.getDataLayout();
7527 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
7528 Align TempAlign = DAG.getEVTAlign(EnvVT);
7529 SDValue Chain = getRoot();
7530 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7531 // and temporary storage in stack.
7533 Res = DAG.getNode(
7534 ISD::GET_FPENV, sdl,
7535 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7536 MVT::Other),
7537 Chain);
7538 } else {
7539 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7540 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7541 auto MPI =
7542 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
7543 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7545 TempAlign);
7546 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7547 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
7548 }
7549 setValue(&I, Res);
7550 DAG.setRoot(Res.getValue(1));
7551 return;
7552 }
7553 case Intrinsic::set_fpenv: {
7554 SDValue Env = getValue(I.getArgOperand(0));
7555 EVT EnvVT = Env.getValueType();
7556 Align TempAlign = DAG.getEVTAlign(EnvVT);
7557 SDValue Chain = getRoot();
7558 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7559 // environment from memory.
7561 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
7562 } else {
7563 // Allocate space in stack, copy environment bits into it and use this
7564 // memory in SET_FPENV_MEM.
7565 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7566 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7567 auto MPI =
7568 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
7569 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7571 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7573 TempAlign);
7574 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7575 }
7576 DAG.setRoot(Chain);
7577 return;
7578 }
7579 case Intrinsic::reset_fpenv:
7580 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
7581 return;
7582 case Intrinsic::get_fpmode:
7583 Res = DAG.getNode(
7584 ISD::GET_FPMODE, sdl,
7585 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7586 MVT::Other),
7587 DAG.getRoot());
7588 setValue(&I, Res);
7589 DAG.setRoot(Res.getValue(1));
7590 return;
7591 case Intrinsic::set_fpmode:
7592 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
7593 getValue(I.getArgOperand(0)));
7594 DAG.setRoot(Res);
7595 return;
7596 case Intrinsic::reset_fpmode: {
7597 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
7598 DAG.setRoot(Res);
7599 return;
7600 }
7601 case Intrinsic::pcmarker: {
7602 SDValue Tmp = getValue(I.getArgOperand(0));
7603 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
7604 return;
7605 }
7606 case Intrinsic::readcyclecounter: {
7607 SDValue Op = getRoot();
7608 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
7609 DAG.getVTList(MVT::i64, MVT::Other), Op);
7610 setValue(&I, Res);
7611 DAG.setRoot(Res.getValue(1));
7612 return;
7613 }
7614 case Intrinsic::readsteadycounter: {
7615 SDValue Op = getRoot();
7616 Res = DAG.getNode(ISD::READSTEADYCOUNTER, sdl,
7617 DAG.getVTList(MVT::i64, MVT::Other), Op);
7618 setValue(&I, Res);
7619 DAG.setRoot(Res.getValue(1));
7620 return;
7621 }
7622 case Intrinsic::bitreverse:
7623 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
7624 getValue(I.getArgOperand(0)).getValueType(),
7625 getValue(I.getArgOperand(0))));
7626 return;
7627 case Intrinsic::bswap:
7628 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
7629 getValue(I.getArgOperand(0)).getValueType(),
7630 getValue(I.getArgOperand(0))));
7631 return;
7632 case Intrinsic::cttz: {
7633 SDValue Arg = getValue(I.getArgOperand(0));
7634 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7635 EVT Ty = Arg.getValueType();
7636 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_POISON,
7637 sdl, Ty, Arg));
7638 return;
7639 }
7640 case Intrinsic::ctlz: {
7641 SDValue Arg = getValue(I.getArgOperand(0));
7642 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7643 EVT Ty = Arg.getValueType();
7644 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_POISON,
7645 sdl, Ty, Arg));
7646 return;
7647 }
7648 case Intrinsic::ctpop: {
7649 SDValue Arg = getValue(I.getArgOperand(0));
7650 EVT Ty = Arg.getValueType();
7651 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7652 return;
7653 }
7654 case Intrinsic::fshl:
7655 case Intrinsic::fshr: {
7656 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7657 SDValue X = getValue(I.getArgOperand(0));
7658 SDValue Y = getValue(I.getArgOperand(1));
7659 SDValue Z = getValue(I.getArgOperand(2));
7660 EVT VT = X.getValueType();
7661
7662 if (X == Y) {
7663 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7664 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7665 } else {
7666 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7667 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7668 }
7669 return;
7670 }
7671 case Intrinsic::clmul: {
7672 SDValue X = getValue(I.getArgOperand(0));
7673 SDValue Y = getValue(I.getArgOperand(1));
7674 setValue(&I, DAG.getNode(ISD::CLMUL, sdl, X.getValueType(), X, Y));
7675 return;
7676 }
7677 case Intrinsic::pext: {
7678 SDValue X = getValue(I.getArgOperand(0));
7679 SDValue Y = getValue(I.getArgOperand(1));
7680 setValue(&I, DAG.getNode(ISD::PEXT, sdl, X.getValueType(), X, Y));
7681 return;
7682 }
7683 case Intrinsic::pdep: {
7684 SDValue X = getValue(I.getArgOperand(0));
7685 SDValue Y = getValue(I.getArgOperand(1));
7686 setValue(&I, DAG.getNode(ISD::PDEP, sdl, X.getValueType(), X, Y));
7687 return;
7688 }
7689 case Intrinsic::smulh:
7690 case Intrinsic::umulh: {
7691 auto Opc = Intrinsic == Intrinsic::smulh ? ISD::MULHS : ISD::MULHU;
7692 SDValue X = getValue(I.getArgOperand(0));
7693 SDValue Y = getValue(I.getArgOperand(1));
7694 setValue(&I, DAG.getNode(Opc, sdl, X.getValueType(), X, Y));
7695 return;
7696 }
7697 case Intrinsic::sadd_sat: {
7698 SDValue Op1 = getValue(I.getArgOperand(0));
7699 SDValue Op2 = getValue(I.getArgOperand(1));
7700 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7701 return;
7702 }
7703 case Intrinsic::uadd_sat: {
7704 SDValue Op1 = getValue(I.getArgOperand(0));
7705 SDValue Op2 = getValue(I.getArgOperand(1));
7706 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7707 return;
7708 }
7709 case Intrinsic::ssub_sat: {
7710 SDValue Op1 = getValue(I.getArgOperand(0));
7711 SDValue Op2 = getValue(I.getArgOperand(1));
7712 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7713 return;
7714 }
7715 case Intrinsic::usub_sat: {
7716 SDValue Op1 = getValue(I.getArgOperand(0));
7717 SDValue Op2 = getValue(I.getArgOperand(1));
7718 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7719 return;
7720 }
7721 case Intrinsic::sshl_sat:
7722 case Intrinsic::ushl_sat: {
7723 SDValue Op1 = getValue(I.getArgOperand(0));
7724 SDValue Op2 = getValue(I.getArgOperand(1));
7725
7726 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
7727 Op1.getValueType(), DAG.getDataLayout());
7728
7729 // Coerce the shift amount to the right type if we can. This exposes the
7730 // truncate or zext to optimization early.
7731 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
7732 assert(ShiftTy.getSizeInBits() >=
7734 "Unexpected shift type");
7735 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
7736 }
7737
7738 unsigned Opc =
7739 Intrinsic == Intrinsic::sshl_sat ? ISD::SSHLSAT : ISD::USHLSAT;
7740 setValue(&I, DAG.getNode(Opc, sdl, Op1.getValueType(), Op1, Op2));
7741 return;
7742 }
7743 case Intrinsic::smul_fix:
7744 case Intrinsic::umul_fix:
7745 case Intrinsic::smul_fix_sat:
7746 case Intrinsic::umul_fix_sat: {
7747 SDValue Op1 = getValue(I.getArgOperand(0));
7748 SDValue Op2 = getValue(I.getArgOperand(1));
7749 SDValue Op3 = getValue(I.getArgOperand(2));
7750 setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
7751 Op1.getValueType(), Op1, Op2, Op3));
7752 return;
7753 }
7754 case Intrinsic::sdiv_fix:
7755 case Intrinsic::udiv_fix:
7756 case Intrinsic::sdiv_fix_sat:
7757 case Intrinsic::udiv_fix_sat: {
7758 SDValue Op1 = getValue(I.getArgOperand(0));
7759 SDValue Op2 = getValue(I.getArgOperand(1));
7760 SDValue Op3 = getValue(I.getArgOperand(2));
7762 Op1, Op2, Op3, DAG, TLI));
7763 return;
7764 }
7765 case Intrinsic::smax: {
7766 SDValue Op1 = getValue(I.getArgOperand(0));
7767 SDValue Op2 = getValue(I.getArgOperand(1));
7768 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7769 return;
7770 }
7771 case Intrinsic::smin: {
7772 SDValue Op1 = getValue(I.getArgOperand(0));
7773 SDValue Op2 = getValue(I.getArgOperand(1));
7774 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7775 return;
7776 }
7777 case Intrinsic::umax: {
7778 SDValue Op1 = getValue(I.getArgOperand(0));
7779 SDValue Op2 = getValue(I.getArgOperand(1));
7780 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7781 return;
7782 }
7783 case Intrinsic::umin: {
7784 SDValue Op1 = getValue(I.getArgOperand(0));
7785 SDValue Op2 = getValue(I.getArgOperand(1));
7786 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7787 return;
7788 }
7789 case Intrinsic::abs: {
7790 SDValue Op1 = getValue(I.getArgOperand(0));
7791 bool IntMinIsPoison = cast<ConstantInt>(I.getArgOperand(1))->isOne();
7792 unsigned Opc = IntMinIsPoison ? ISD::ABS_MIN_POISON : ISD::ABS;
7793 setValue(&I, DAG.getNode(Opc, sdl, Op1.getValueType(), Op1));
7794 return;
7795 }
7796 case Intrinsic::scmp: {
7797 SDValue Op1 = getValue(I.getArgOperand(0));
7798 SDValue Op2 = getValue(I.getArgOperand(1));
7799 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7800 setValue(&I, DAG.getNode(ISD::SCMP, sdl, DestVT, Op1, Op2));
7801 break;
7802 }
7803 case Intrinsic::ucmp: {
7804 SDValue Op1 = getValue(I.getArgOperand(0));
7805 SDValue Op2 = getValue(I.getArgOperand(1));
7806 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7807 setValue(&I, DAG.getNode(ISD::UCMP, sdl, DestVT, Op1, Op2));
7808 break;
7809 }
7810 case Intrinsic::stackaddress:
7811 case Intrinsic::stacksave: {
7812 unsigned SDOpcode = Intrinsic == Intrinsic::stackaddress ? ISD::STACKADDRESS
7814 SDValue Op = getRoot();
7815 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7816 Res = DAG.getNode(SDOpcode, sdl, DAG.getVTList(VT, MVT::Other), Op);
7817 setValue(&I, Res);
7818 DAG.setRoot(Res.getValue(1));
7819 return;
7820 }
7821 case Intrinsic::stackrestore:
7822 Res = getValue(I.getArgOperand(0));
7823 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7824 return;
7825 case Intrinsic::get_dynamic_area_offset: {
7826 SDValue Op = getRoot();
7827 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7828 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
7829 Op);
7830 DAG.setRoot(Op);
7831 setValue(&I, Res);
7832 return;
7833 }
7834 case Intrinsic::stackguard: {
7835 MachineFunction &MF = DAG.getMachineFunction();
7836 const Module &M = *MF.getFunction().getParent();
7837 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7838 SDValue Chain = getRoot();
7839 if (TLI.useLoadStackGuardNode(M)) {
7840 Res = getLoadStackGuard(DAG, sdl, Chain);
7841 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7842 } else {
7843 const Value *Global = TLI.getSDagStackGuard(M, DAG.getLibcalls());
7844 if (!Global) {
7845 LLVMContext &Ctx = *DAG.getContext();
7846 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
7847 setValue(&I, DAG.getPOISON(PtrTy));
7848 return;
7849 }
7850
7851 Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
7852 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7853 MachinePointerInfo(Global, 0), Align,
7855 }
7856 // Mix the cookie with FP if enabled. Skip if using LOAD_STACK_GUARD
7857 // with post-RA mixing (AArch64 MSVCRT), as the mixing will be done during
7858 // post-RA expansion of LOAD_STACK_GUARD.
7860 Res = TLI.emitStackGuardMixFP(DAG, Res, sdl);
7861 DAG.setRoot(Chain);
7862 setValue(&I, Res);
7863 return;
7864 }
7865 case Intrinsic::stackprotector: {
7866 // Emit code into the DAG to store the stack guard onto the stack.
7867 MachineFunction &MF = DAG.getMachineFunction();
7868 MachineFrameInfo &MFI = MF.getFrameInfo();
7869 const Module &M = *MF.getFunction().getParent();
7870 SDValue Src, Chain = getRoot();
7871
7873 Src = getLoadStackGuard(DAG, sdl, Chain);
7874 else
7875 Src = getValue(I.getArgOperand(0)); // The guard's value.
7876
7877 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7878
7879 int FI = FuncInfo.StaticAllocaMap[Slot];
7880 MFI.setStackProtectorIndex(FI);
7881 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7882
7883 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7884
7885 // Store the stack protector onto the stack.
7886 Res = DAG.getStore(
7887 Chain, sdl, Src, FIN,
7888 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
7889 MaybeAlign(), MachineMemOperand::MOVolatile);
7890 setValue(&I, Res);
7891 DAG.setRoot(Res);
7892 return;
7893 }
7894 case Intrinsic::objectsize:
7895 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7896
7897 case Intrinsic::is_constant:
7898 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7899
7900 case Intrinsic::annotation:
7901 case Intrinsic::ptr_annotation:
7902 case Intrinsic::launder_invariant_group:
7903 // Drop the intrinsic, but forward the value
7904 setValue(&I, getValue(I.getOperand(0)));
7905 return;
7906
7907 case Intrinsic::type_test:
7908 case Intrinsic::public_type_test:
7909 case Intrinsic::type_checked_load:
7910 case Intrinsic::type_checked_load_relative: {
7911 // These intrinsics are expected to be lowered by the LowerTypeTests pass
7912 // before code generation. Surviving until here usually indicates a
7913 // misconfiguration, for instance when devirtualization is enabled but LTO
7914 // does not actually run.
7915 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
7916 *I.getFunction(),
7917 Intrinsic::getBaseName(Intrinsic) +
7918 " intrinsic must be lowered by the LowerTypeTests pass "
7919 "before code generation",
7920 sdl.getDebugLoc()));
7921
7922 // Lower the result to poison so that compilation can continue and collect
7923 // any further diagnostics.
7924 setValueToPoison(&I, sdl);
7925 return;
7926 }
7927
7928 case Intrinsic::assume:
7929 case Intrinsic::experimental_noalias_scope_decl:
7930 case Intrinsic::var_annotation:
7931 case Intrinsic::sideeffect:
7932 // Discard annotate attributes, noalias scope declarations, assumptions, and
7933 // artificial side-effects.
7934 return;
7935
7936 case Intrinsic::codeview_annotation: {
7937 // Emit a label associated with this metadata.
7938 MachineFunction &MF = DAG.getMachineFunction();
7939 MCSymbol *Label = MF.getContext().createTempSymbol("annotation", true);
7940 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7941 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7942 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7943 DAG.setRoot(Res);
7944 return;
7945 }
7946
7947 case Intrinsic::init_trampoline: {
7948 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7949
7950 SDValue Ops[6];
7951 Ops[0] = getRoot();
7952 Ops[1] = getValue(I.getArgOperand(0));
7953 Ops[2] = getValue(I.getArgOperand(1));
7954 Ops[3] = getValue(I.getArgOperand(2));
7955 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7956 Ops[5] = DAG.getSrcValue(F);
7957
7958 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7959
7960 DAG.setRoot(Res);
7961 return;
7962 }
7963 case Intrinsic::adjust_trampoline:
7964 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
7965 TLI.getPointerTy(DAG.getDataLayout()),
7966 getValue(I.getArgOperand(0))));
7967 return;
7968 case Intrinsic::gcroot: {
7969 assert(DAG.getMachineFunction().getFunction().hasGC() &&
7970 "only valid in functions with gc specified, enforced by Verifier");
7971 assert(GFI && "implied by previous");
7972 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7973 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7974
7975 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7976 GFI->addStackRoot(FI->getIndex(), TypeMap);
7977 return;
7978 }
7979 case Intrinsic::gcread:
7980 case Intrinsic::gcwrite:
7981 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7982 case Intrinsic::get_rounding:
7983 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7984 setValue(&I, Res);
7985 DAG.setRoot(Res.getValue(1));
7986 return;
7987
7988 case Intrinsic::expect:
7989 case Intrinsic::expect_with_probability:
7990 // Just replace __builtin_expect(exp, c) and
7991 // __builtin_expect_with_probability(exp, c, p) with EXP.
7992 setValue(&I, getValue(I.getArgOperand(0)));
7993 return;
7994
7995 case Intrinsic::ubsantrap:
7996 case Intrinsic::debugtrap:
7997 case Intrinsic::trap: {
7998 StringRef TrapFuncName =
7999 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
8000 if (TrapFuncName.empty()) {
8001 switch (Intrinsic) {
8002 case Intrinsic::trap:
8003 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
8004 break;
8005 case Intrinsic::debugtrap:
8006 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
8007 break;
8008 case Intrinsic::ubsantrap:
8009 DAG.setRoot(DAG.getNode(
8010 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
8011 DAG.getTargetConstant(
8012 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
8013 MVT::i32)));
8014 break;
8015 default: llvm_unreachable("unknown trap intrinsic");
8016 }
8017 DAG.addNoMergeSiteInfo(DAG.getRoot().getNode(),
8018 I.hasFnAttr(Attribute::NoMerge));
8019 return;
8020 }
8022 if (Intrinsic == Intrinsic::ubsantrap) {
8023 Value *Arg = I.getArgOperand(0);
8024 Args.emplace_back(Arg, getValue(Arg),
8025 LibInfo->getExtAttrForI8Param(/*Signed=*/false));
8026 }
8027
8028 TargetLowering::CallLoweringInfo CLI(DAG);
8029 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
8030 CallingConv::C, I.getType(),
8031 DAG.getExternalSymbol(TrapFuncName.data(),
8032 TLI.getPointerTy(DAG.getDataLayout())),
8033 std::move(Args));
8034 CLI.NoMerge = I.hasFnAttr(Attribute::NoMerge);
8035 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8036 DAG.setRoot(Result.second);
8037 return;
8038 }
8039
8040 case Intrinsic::allow_runtime_check:
8041 case Intrinsic::allow_ubsan_check:
8042 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
8043 return;
8044
8045 case Intrinsic::uadd_with_overflow:
8046 case Intrinsic::sadd_with_overflow:
8047 case Intrinsic::usub_with_overflow:
8048 case Intrinsic::ssub_with_overflow:
8049 case Intrinsic::umul_with_overflow:
8050 case Intrinsic::smul_with_overflow: {
8052 switch (Intrinsic) {
8053 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8054 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
8055 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
8056 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
8057 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
8058 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
8059 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
8060 }
8061 SDValue Op1 = getValue(I.getArgOperand(0));
8062 SDValue Op2 = getValue(I.getArgOperand(1));
8063
8064 EVT ResultVT = Op1.getValueType();
8065 EVT OverflowVT = ResultVT.changeElementType(*Context, MVT::i1);
8066
8067 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
8068 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
8069 return;
8070 }
8071 case Intrinsic::prefetch: {
8072 SDValue Ops[5];
8073 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
8075 Ops[0] = DAG.getRoot();
8076 Ops[1] = getValue(I.getArgOperand(0));
8077 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
8078 MVT::i32);
8079 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
8080 MVT::i32);
8081 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
8082 MVT::i32);
8083 SDValue Result = DAG.getMemIntrinsicNode(
8084 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
8085 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
8086 /* align */ std::nullopt, Flags);
8087
8088 // Chain the prefetch in parallel with any pending loads, to stay out of
8089 // the way of later optimizations.
8090 PendingLoads.push_back(Result);
8091 Result = getRoot();
8092 DAG.setRoot(Result);
8093 return;
8094 }
8095 case Intrinsic::lifetime_start:
8096 case Intrinsic::lifetime_end: {
8097 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
8098 // Stack coloring is not enabled in O0, discard region information.
8099 if (TM.getOptLevel() == CodeGenOptLevel::None)
8100 return;
8101
8102 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(I.getArgOperand(0));
8103 if (!LifetimeObject)
8104 return;
8105
8106 // First check that the Alloca is static, otherwise it won't have a
8107 // valid frame index.
8108 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
8109 if (SI == FuncInfo.StaticAllocaMap.end())
8110 return;
8111
8112 const int FrameIndex = SI->second;
8113 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex);
8114 DAG.setRoot(Res);
8115 return;
8116 }
8117 case Intrinsic::pseudoprobe: {
8118 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
8119 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
8120 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
8121 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
8122 DAG.setRoot(Res);
8123 return;
8124 }
8125 case Intrinsic::invariant_start:
8126 // Discard region information.
8127 setValue(&I,
8128 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
8129 return;
8130 case Intrinsic::invariant_end:
8131 // Discard region information.
8132 return;
8133 case Intrinsic::clear_cache: {
8134 SDValue InputChain = DAG.getRoot();
8135 SDValue StartVal = getValue(I.getArgOperand(0));
8136 SDValue EndVal = getValue(I.getArgOperand(1));
8137 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),
8138 {InputChain, StartVal, EndVal});
8139 setValue(&I, Res);
8140 DAG.setRoot(Res);
8141 return;
8142 }
8143 case Intrinsic::donothing:
8144 case Intrinsic::seh_try_begin:
8145 case Intrinsic::seh_scope_begin:
8146 case Intrinsic::seh_try_end:
8147 case Intrinsic::seh_scope_end:
8148 // ignore
8149 return;
8150 case Intrinsic::experimental_stackmap:
8151 visitStackmap(I);
8152 return;
8153 case Intrinsic::experimental_patchpoint_void:
8154 case Intrinsic::experimental_patchpoint:
8155 visitPatchpoint(I);
8156 return;
8157 case Intrinsic::experimental_gc_statepoint:
8159 return;
8160 case Intrinsic::experimental_gc_result:
8161 visitGCResult(cast<GCResultInst>(I));
8162 return;
8163 case Intrinsic::experimental_gc_relocate:
8164 visitGCRelocate(cast<GCRelocateInst>(I));
8165 return;
8166 case Intrinsic::instrprof_cover:
8167 llvm_unreachable("instrprof failed to lower a cover");
8168 case Intrinsic::instrprof_increment:
8169 llvm_unreachable("instrprof failed to lower an increment");
8170 case Intrinsic::instrprof_timestamp:
8171 llvm_unreachable("instrprof failed to lower a timestamp");
8172 case Intrinsic::instrprof_value_profile:
8173 llvm_unreachable("instrprof failed to lower a value profiling call");
8174 case Intrinsic::instrprof_mcdc_parameters:
8175 llvm_unreachable("instrprof failed to lower mcdc parameters");
8176 case Intrinsic::instrprof_mcdc_tvbitmap_update:
8177 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
8178 case Intrinsic::localescape: {
8179 MachineFunction &MF = DAG.getMachineFunction();
8180 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
8181
8182 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
8183 // is the same on all targets.
8184 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
8185 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
8186 if (isa<ConstantPointerNull>(Arg))
8187 continue; // Skip null pointers. They represent a hole in index space.
8188 AllocaInst *Slot = cast<AllocaInst>(Arg);
8189 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
8190 "can only escape static allocas");
8191 int FI = FuncInfo.StaticAllocaMap[Slot];
8192 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8194 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
8195 TII->get(TargetOpcode::LOCAL_ESCAPE))
8196 .addSym(FrameAllocSym)
8197 .addFrameIndex(FI);
8198 }
8199
8200 return;
8201 }
8202
8203 case Intrinsic::localrecover: {
8204 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
8205 MachineFunction &MF = DAG.getMachineFunction();
8206
8207 // Get the symbol that defines the frame offset.
8208 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
8209 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
8210 unsigned IdxVal =
8211 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
8212 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8214
8215 Value *FP = I.getArgOperand(1);
8216 SDValue FPVal = getValue(FP);
8217 EVT PtrVT = FPVal.getValueType();
8218
8219 // Create a MCSymbol for the label to avoid any target lowering
8220 // that would make this PC relative.
8221 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
8222 SDValue OffsetVal =
8223 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
8224
8225 // Add the offset to the FP.
8226 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
8227 setValue(&I, Add);
8228
8229 return;
8230 }
8231
8232 case Intrinsic::fake_use: {
8233 Value *V = I.getArgOperand(0);
8234 SDValue Ops[2];
8235 // For Values not declared or previously used in this basic block, the
8236 // NodeMap will not have an entry, and `getValue` will assert if V has no
8237 // valid register value.
8238 auto FakeUseValue = [&]() -> SDValue {
8239 SDValue &N = NodeMap[V];
8240 if (N.getNode())
8241 return N;
8242
8243 // If there's a virtual register allocated and initialized for this
8244 // value, use it.
8245 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
8246 return copyFromReg;
8247 // FIXME: Do we want to preserve constants? It seems pointless.
8248 if (isa<Constant>(V))
8249 return getValue(V);
8250 return SDValue();
8251 }();
8252 if (!FakeUseValue || FakeUseValue.isUndef())
8253 return;
8254 Ops[0] = getRoot();
8255 Ops[1] = FakeUseValue;
8256 // Also, do not translate a fake use with an undef operand, or any other
8257 // empty SDValues.
8258 if (!Ops[1] || Ops[1].isUndef())
8259 return;
8260 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other, Ops));
8261 return;
8262 }
8263
8264 case Intrinsic::reloc_none: {
8265 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
8266 StringRef SymbolName = cast<MDString>(MD)->getString();
8267 SDValue Ops[2] = {
8268 getRoot(),
8269 DAG.getTargetExternalSymbol(
8270 SymbolName.data(), TLI.getProgramPointerTy(DAG.getDataLayout()))};
8271 DAG.setRoot(DAG.getNode(ISD::RELOC_NONE, sdl, MVT::Other, Ops));
8272 return;
8273 }
8274
8275 case Intrinsic::cond_loop: {
8276 SDValue InputChain = DAG.getRoot();
8277 SDValue P = getValue(I.getArgOperand(0));
8278 Res = DAG.getNode(ISD::COND_LOOP, sdl, DAG.getVTList(MVT::Other),
8279 {InputChain, P});
8280 setValue(&I, Res);
8281 DAG.setRoot(Res);
8282 return;
8283 }
8284
8285 case Intrinsic::eh_exceptionpointer:
8286 case Intrinsic::eh_exceptioncode: {
8287 // Get the exception pointer vreg, copy from it, and resize it to fit.
8288 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
8289 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
8290 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
8291 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
8292 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
8293 if (Intrinsic == Intrinsic::eh_exceptioncode)
8294 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
8295 setValue(&I, N);
8296 return;
8297 }
8298 case Intrinsic::xray_customevent: {
8299 // Here we want to make sure that the intrinsic behaves as if it has a
8300 // specific calling convention.
8301 const auto &Triple = DAG.getTarget().getTargetTriple();
8302 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64 &&
8303 Triple.getArch() != Triple::hexagon)
8304 return;
8305
8307
8308 // We want to say that we always want the arguments in registers.
8309 SDValue LogEntryVal = getValue(I.getArgOperand(0));
8310 SDValue StrSizeVal = getValue(I.getArgOperand(1));
8311 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8312 SDValue Chain = getRoot();
8313 Ops.push_back(LogEntryVal);
8314 Ops.push_back(StrSizeVal);
8315 Ops.push_back(Chain);
8316
8317 // We need to enforce the calling convention for the callsite, so that
8318 // argument ordering is enforced correctly, and that register allocation can
8319 // see that some registers may be assumed clobbered and have to preserve
8320 // them across calls to the intrinsic.
8321 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
8322 sdl, NodeTys, Ops);
8323 SDValue patchableNode = SDValue(MN, 0);
8324 DAG.setRoot(patchableNode);
8325 setValue(&I, patchableNode);
8326 return;
8327 }
8328 case Intrinsic::xray_typedevent: {
8329 // Here we want to make sure that the intrinsic behaves as if it has a
8330 // specific calling convention.
8331 const auto &Triple = DAG.getTarget().getTargetTriple();
8332 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64 &&
8333 Triple.getArch() != Triple::hexagon)
8334 return;
8335
8337
8338 // We want to say that we always want the arguments in registers.
8339 // It's unclear to me how manipulating the selection DAG here forces callers
8340 // to provide arguments in registers instead of on the stack.
8341 SDValue LogTypeId = getValue(I.getArgOperand(0));
8342 SDValue LogEntryVal = getValue(I.getArgOperand(1));
8343 SDValue StrSizeVal = getValue(I.getArgOperand(2));
8344 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8345 SDValue Chain = getRoot();
8346 Ops.push_back(LogTypeId);
8347 Ops.push_back(LogEntryVal);
8348 Ops.push_back(StrSizeVal);
8349 Ops.push_back(Chain);
8350
8351 // We need to enforce the calling convention for the callsite, so that
8352 // argument ordering is enforced correctly, and that register allocation can
8353 // see that some registers may be assumed clobbered and have to preserve
8354 // them across calls to the intrinsic.
8355 MachineSDNode *MN = DAG.getMachineNode(
8356 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
8357 SDValue patchableNode = SDValue(MN, 0);
8358 DAG.setRoot(patchableNode);
8359 setValue(&I, patchableNode);
8360 return;
8361 }
8362 case Intrinsic::experimental_deoptimize:
8364 return;
8365 case Intrinsic::stepvector:
8366 visitStepVector(I);
8367 return;
8368 case Intrinsic::vector_reduce_fadd:
8369 case Intrinsic::vector_reduce_fmul:
8370 case Intrinsic::vector_reduce_add:
8371 case Intrinsic::vector_reduce_mul:
8372 case Intrinsic::vector_reduce_and:
8373 case Intrinsic::vector_reduce_or:
8374 case Intrinsic::vector_reduce_xor:
8375 case Intrinsic::vector_reduce_smax:
8376 case Intrinsic::vector_reduce_smin:
8377 case Intrinsic::vector_reduce_umax:
8378 case Intrinsic::vector_reduce_umin:
8379 case Intrinsic::vector_reduce_fmax:
8380 case Intrinsic::vector_reduce_fmin:
8381 case Intrinsic::vector_reduce_fmaximum:
8382 case Intrinsic::vector_reduce_fminimum:
8383 case Intrinsic::vector_reduce_fmaximumnum:
8384 case Intrinsic::vector_reduce_fminimumnum:
8385 visitVectorReduce(I, Intrinsic);
8386 return;
8387
8388 case Intrinsic::icall_branch_funnel: {
8390 Ops.push_back(getValue(I.getArgOperand(0)));
8391
8392 int64_t Offset;
8394 I.getArgOperand(1), Offset, DAG.getDataLayout()));
8395 if (!Base)
8397 "llvm.icall.branch.funnel operand must be a GlobalValue");
8398 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
8399
8400 struct BranchFunnelTarget {
8401 int64_t Offset;
8403 };
8405
8406 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
8408 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
8409 if (ElemBase != Base)
8410 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
8411 "to the same GlobalValue");
8412
8413 SDValue Val = getValue(I.getArgOperand(Op + 1));
8414 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
8415 if (!GA)
8417 "llvm.icall.branch.funnel operand must be a GlobalValue");
8418 Targets.push_back({Offset, DAG.getTargetGlobalAddress(
8419 GA->getGlobal(), sdl, Val.getValueType(),
8420 GA->getOffset())});
8421 }
8422 llvm::sort(Targets,
8423 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
8424 return T1.Offset < T2.Offset;
8425 });
8426
8427 for (auto &T : Targets) {
8428 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
8429 Ops.push_back(T.Target);
8430 }
8431
8432 Ops.push_back(DAG.getRoot()); // Chain
8433 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
8434 MVT::Other, Ops),
8435 0);
8436 DAG.setRoot(N);
8437 setValue(&I, N);
8438 HasTailCall = true;
8439 return;
8440 }
8441
8442 case Intrinsic::wasm_landingpad_index:
8443 // Information this intrinsic contained has been transferred to
8444 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
8445 // delete it now.
8446 return;
8447
8448 case Intrinsic::aarch64_settag:
8449 case Intrinsic::aarch64_settag_zero: {
8450 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8451 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
8453 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
8454 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
8455 ZeroMemory);
8456 DAG.setRoot(Val);
8457 setValue(&I, Val);
8458 return;
8459 }
8460 case Intrinsic::amdgcn_cs_chain: {
8461 // At this point we don't care if it's amdgpu_cs_chain or
8462 // amdgpu_cs_chain_preserve.
8464
8465 Type *RetTy = I.getType();
8466 assert(RetTy->isVoidTy() && "Should not return");
8467
8468 SDValue Callee = getValue(I.getOperand(0));
8469
8470 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
8471 // We'll also tack the value of the EXEC mask at the end.
8473 Args.reserve(3);
8474
8475 for (unsigned Idx : {2, 3, 1}) {
8476 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
8477 I.getOperand(Idx)->getType());
8478 Arg.setAttributes(&I, Idx);
8479 Args.push_back(Arg);
8480 }
8481
8482 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
8483 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
8484 Args[2].IsInReg = true; // EXEC should be inreg
8485
8486 // Forward the flags and any additional arguments.
8487 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
8488 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
8489 I.getOperand(Idx)->getType());
8490 Arg.setAttributes(&I, Idx);
8491 Args.push_back(Arg);
8492 }
8493
8494 TargetLowering::CallLoweringInfo CLI(DAG);
8495 CLI.setDebugLoc(getCurSDLoc())
8496 .setChain(getRoot())
8497 .setCallee(CC, RetTy, Callee, std::move(Args))
8498 .setNoReturn(true)
8499 .setTailCall(true)
8500 .setConvergent(I.isConvergent());
8501 CLI.CB = &I;
8502 std::pair<SDValue, SDValue> Result =
8503 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
8504 (void)Result;
8505 assert(!Result.first.getNode() && !Result.second.getNode() &&
8506 "Should've lowered as tail call");
8507
8508 HasTailCall = true;
8509 return;
8510 }
8511 case Intrinsic::amdgcn_call_whole_wave: {
8513 bool isTailCall = I.isTailCall();
8514
8515 // The first argument is the callee. Skip it when assembling the call args.
8516 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
8517 TargetLowering::ArgListEntry Arg(getValue(I.getArgOperand(Idx)),
8518 I.getArgOperand(Idx)->getType());
8519 Arg.setAttributes(&I, Idx);
8520
8521 // If we have an explicit sret argument that is an Instruction, (i.e., it
8522 // might point to function-local memory), we can't meaningfully tail-call.
8523 if (Arg.IsSRet && isa<Instruction>(I.getArgOperand(Idx)))
8524 isTailCall = false;
8525
8526 Args.push_back(Arg);
8527 }
8528
8529 SDValue ConvControlToken;
8530 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8531 auto *Token = Bundle->Inputs[0].get();
8532 ConvControlToken = getValue(Token);
8533 }
8534
8535 TargetLowering::CallLoweringInfo CLI(DAG);
8536 CLI.setDebugLoc(getCurSDLoc())
8537 .setChain(getRoot())
8538 .setCallee(CallingConv::AMDGPU_Gfx_WholeWave, I.getType(),
8539 getValue(I.getArgOperand(0)), std::move(Args))
8540 .setTailCall(isTailCall && canTailCall(I))
8541 .setIsPreallocated(
8542 I.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
8543 .setConvergent(I.isConvergent())
8544 .setConvergenceControlToken(ConvControlToken);
8545 CLI.CB = &I;
8546
8547 std::pair<SDValue, SDValue> Result =
8548 lowerInvokable(CLI, /*EHPadBB=*/nullptr);
8549
8550 if (Result.first.getNode())
8551 setValue(&I, Result.first);
8552 return;
8553 }
8554 case Intrinsic::ptrmask: {
8555 SDValue Ptr = getValue(I.getOperand(0));
8556 SDValue Mask = getValue(I.getOperand(1));
8557
8558 // On arm64_32, pointers are 32 bits when stored in memory, but
8559 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
8560 // match the index type, but the pointer is 64 bits, so the mask must be
8561 // zero-extended up to 64 bits to match the pointer.
8562 EVT PtrVT =
8563 TLI.getValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8564 EVT MemVT =
8565 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8566 assert(PtrVT == Ptr.getValueType());
8567 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
8568 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
8569 // 128-bit, so we have to pad the mask with ones for unused bits.
8570 auto HighOnes = DAG.getNode(
8571 ISD::SHL, sdl, PtrVT, DAG.getAllOnesConstant(sdl, PtrVT),
8572 DAG.getShiftAmountConstant(Mask.getValueType().getFixedSizeInBits(),
8573 PtrVT, sdl));
8574 Mask = DAG.getNode(ISD::OR, sdl, PtrVT,
8575 DAG.getZExtOrTrunc(Mask, sdl, PtrVT), HighOnes);
8576 } else if (Mask.getValueType() != PtrVT)
8577 Mask = DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);
8578
8579 assert(Mask.getValueType() == PtrVT);
8580 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
8581 return;
8582 }
8583 case Intrinsic::threadlocal_address: {
8584 setValue(&I, getValue(I.getOperand(0)));
8585 return;
8586 }
8587 case Intrinsic::get_active_lane_mask: {
8588 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8589 SDValue Index = getValue(I.getOperand(0));
8590 SDValue TripCount = getValue(I.getOperand(1));
8591 EVT ElementVT = Index.getValueType();
8592
8593 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
8594 setValue(&I, DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, sdl, CCVT, Index,
8595 TripCount));
8596 return;
8597 }
8598
8599 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
8600 CCVT.getVectorElementCount());
8601
8602 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
8603 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
8604 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
8605 SDValue VectorInduction = DAG.getNode(
8606 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
8607 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
8608 VectorTripCount, ISD::CondCode::SETULT);
8609 setValue(&I, SetCC);
8610 return;
8611 }
8612 case Intrinsic::mask_beforefirst: {
8613 SDValue Op = getValue(I.getOperand(0));
8614 setValue(&I,
8615 DAG.getNode(ISD::MASK_BEFOREFIRST, sdl, Op.getValueType(), Op));
8616 return;
8617 }
8618 case Intrinsic::experimental_get_vector_length: {
8619 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8620 "Expected positive VF");
8621 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
8622 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
8623
8624 SDValue Count = getValue(I.getOperand(0));
8625 EVT CountVT = Count.getValueType();
8626
8627 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8628 visitTargetIntrinsic(I, Intrinsic);
8629 return;
8630 }
8631
8632 // Expand to a umin between the trip count and the maximum elements the type
8633 // can hold.
8634 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8635
8636 // Extend the trip count to at least the result VT.
8637 if (CountVT.bitsLT(VT)) {
8638 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
8639 CountVT = VT;
8640 }
8641
8642 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
8644
8645 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
8646 // Clip to the result type if needed.
8647 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
8648
8649 setValue(&I, Trunc);
8650 return;
8651 }
8652 case Intrinsic::vector_partial_reduce_add: {
8653 SDValue Acc = getValue(I.getOperand(0));
8654 SDValue Input = getValue(I.getOperand(1));
8655 setValue(&I,
8656 DAG.getNode(ISD::PARTIAL_REDUCE_UMLA, sdl, Acc.getValueType(), Acc,
8657 Input, DAG.getConstant(1, sdl, Input.getValueType())));
8658 return;
8659 }
8660 case Intrinsic::vector_partial_reduce_fadd: {
8661 SDNodeFlags Flags;
8662 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
8663 Flags.copyFMF(*FPOp);
8664 SDValue Acc = getValue(I.getOperand(0));
8665 SDValue Input = getValue(I.getOperand(1));
8666 setValue(&I, DAG.getNode(ISD::PARTIAL_REDUCE_FMLA, sdl, Acc.getValueType(),
8667 Acc, Input,
8668 DAG.getConstantFP(1.0, sdl, Input.getValueType()),
8669 Flags));
8670 return;
8671 }
8672 case Intrinsic::experimental_cttz_elts: {
8673 SDValue Op = getValue(I.getOperand(0));
8674 EVT OpVT = Op.getValueType();
8675 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
8676 bool ZeroIsPoison =
8677 !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
8678 if (OpVT.getVectorElementType() != MVT::i1) {
8679 // Compare the input vector elements to zero & use to count trailing
8680 // zeros.
8681 SDValue AllZero = DAG.getConstant(0, sdl, OpVT);
8682 EVT I1OpVT = OpVT.changeVectorElementType(*DAG.getContext(), MVT::i1);
8683 Op = DAG.getSetCC(sdl, I1OpVT, Op, AllZero, ISD::SETNE);
8684 }
8685 setValue(&I, DAG.getNode(ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
8687 sdl, RetTy, Op));
8688 return;
8689 }
8690 case Intrinsic::vector_insert: {
8691 SDValue Vec = getValue(I.getOperand(0));
8692 SDValue SubVec = getValue(I.getOperand(1));
8693 SDValue Index = getValue(I.getOperand(2));
8694
8695 // The intrinsic's index type is i64, but the SDNode requires an index type
8696 // suitable for the target. Convert the index as required.
8697 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8698 if (Index.getValueType() != VectorIdxTy)
8699 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8700
8701 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8702 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
8703 Index));
8704 return;
8705 }
8706 case Intrinsic::vector_extract: {
8707 SDValue Vec = getValue(I.getOperand(0));
8708 SDValue Index = getValue(I.getOperand(1));
8709 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8710
8711 // The intrinsic's index type is i64, but the SDNode requires an index type
8712 // suitable for the target. Convert the index as required.
8713 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8714 if (Index.getValueType() != VectorIdxTy)
8715 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8716
8717 setValue(&I,
8718 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
8719 return;
8720 }
8721 case Intrinsic::experimental_vector_match: {
8722 SDValue Op1 = getValue(I.getOperand(0));
8723 SDValue Op2 = getValue(I.getOperand(1));
8724 SDValue Mask = getValue(I.getOperand(2));
8725 EVT ResVT = Mask.getValueType();
8726 setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
8727 return;
8728 }
8729 case Intrinsic::vector_reverse:
8730 visitVectorReverse(I);
8731 return;
8732 case Intrinsic::vector_splice_left:
8733 case Intrinsic::vector_splice_right:
8734 visitVectorSplice(I);
8735 return;
8736 case Intrinsic::callbr_landingpad:
8737 visitCallBrLandingPad(I);
8738 return;
8739 case Intrinsic::vector_interleave2:
8740 visitVectorInterleave(I, 2);
8741 return;
8742 case Intrinsic::vector_interleave3:
8743 visitVectorInterleave(I, 3);
8744 return;
8745 case Intrinsic::vector_interleave4:
8746 visitVectorInterleave(I, 4);
8747 return;
8748 case Intrinsic::vector_interleave5:
8749 visitVectorInterleave(I, 5);
8750 return;
8751 case Intrinsic::vector_interleave6:
8752 visitVectorInterleave(I, 6);
8753 return;
8754 case Intrinsic::vector_interleave7:
8755 visitVectorInterleave(I, 7);
8756 return;
8757 case Intrinsic::vector_interleave8:
8758 visitVectorInterleave(I, 8);
8759 return;
8760 case Intrinsic::vector_deinterleave2:
8761 visitVectorDeinterleave(I, 2);
8762 return;
8763 case Intrinsic::vector_deinterleave3:
8764 visitVectorDeinterleave(I, 3);
8765 return;
8766 case Intrinsic::vector_deinterleave4:
8767 visitVectorDeinterleave(I, 4);
8768 return;
8769 case Intrinsic::vector_deinterleave5:
8770 visitVectorDeinterleave(I, 5);
8771 return;
8772 case Intrinsic::vector_deinterleave6:
8773 visitVectorDeinterleave(I, 6);
8774 return;
8775 case Intrinsic::vector_deinterleave7:
8776 visitVectorDeinterleave(I, 7);
8777 return;
8778 case Intrinsic::vector_deinterleave8:
8779 visitVectorDeinterleave(I, 8);
8780 return;
8781 case Intrinsic::vector_repeat: {
8782 SDValue Vec = getValue(I.getOperand(0));
8783 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8784 setValue(&I, DAG.getNode(ISD::VECTOR_REPEAT, sdl, ResultVT, Vec));
8785 return;
8786 }
8787 case Intrinsic::experimental_vector_compress:
8788 setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,
8789 getValue(I.getArgOperand(0)).getValueType(),
8790 getValue(I.getArgOperand(0)),
8791 getValue(I.getArgOperand(1)),
8792 getValue(I.getArgOperand(2)), Flags));
8793 return;
8794 case Intrinsic::vector_shuffle:
8795 setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, sdl,
8796 TLI.getValueType(DAG.getDataLayout(), I.getType()),
8797 getValue(I.getArgOperand(0)),
8798 getValue(I.getArgOperand(1)), Flags));
8799 return;
8800 case Intrinsic::experimental_convergence_anchor:
8801 case Intrinsic::experimental_convergence_entry:
8802 case Intrinsic::experimental_convergence_loop:
8803 visitConvergenceControl(I, Intrinsic);
8804 return;
8805 case Intrinsic::experimental_vector_histogram_add: {
8806 visitVectorHistogram(I, Intrinsic);
8807 return;
8808 }
8809 case Intrinsic::experimental_vector_extract_last_active: {
8810 visitVectorExtractLastActive(I, Intrinsic);
8811 return;
8812 }
8813 case Intrinsic::loop_dependence_war_mask:
8814 setValue(&I,
8816 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),
8817 getValue(I.getOperand(1)), getValue(I.getOperand(2)),
8818 DAG.getConstant(0, sdl, MVT::i64)));
8819 return;
8820 case Intrinsic::loop_dependence_raw_mask:
8821 setValue(&I,
8823 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),
8824 getValue(I.getOperand(1)), getValue(I.getOperand(2)),
8825 DAG.getConstant(0, sdl, MVT::i64)));
8826 return;
8827 case Intrinsic::masked_udiv:
8828 setValue(&I,
8829 DAG.getNode(ISD::MASKED_UDIV, sdl, EVT::getEVT(I.getType()),
8830 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8831 getValue(I.getOperand(2))));
8832 return;
8833 case Intrinsic::masked_sdiv:
8834 setValue(&I,
8835 DAG.getNode(ISD::MASKED_SDIV, sdl, EVT::getEVT(I.getType()),
8836 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8837 getValue(I.getOperand(2))));
8838 return;
8839 case Intrinsic::masked_urem:
8840 setValue(&I,
8841 DAG.getNode(ISD::MASKED_UREM, sdl, EVT::getEVT(I.getType()),
8842 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8843 getValue(I.getOperand(2))));
8844 return;
8845 case Intrinsic::masked_srem:
8846 setValue(&I,
8847 DAG.getNode(ISD::MASKED_SREM, sdl, EVT::getEVT(I.getType()),
8848 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8849 getValue(I.getOperand(2))));
8850 return;
8851 }
8852}
8853
8854void SelectionDAGBuilder::pushFPOpOutChain(SDValue Result,
8856 assert(Result.getNode()->getNumValues() == 2);
8857 SDValue OutChain = Result.getValue(1);
8858 assert(OutChain.getValueType() == MVT::Other);
8859
8860 // Instead of updating the root immediately, push the produced chain to the
8861 // appropriate list, deferring the update until the root is requested. In this
8862 // case, the nodes from the lists are chained using TokenFactor, indicating
8863 // that the operations are independent.
8864 //
8865 // In particular, the root is updated before any call that might access the
8866 // floating-point environment, except for constrained intrinsics.
8867 switch (EB) {
8870 PendingConstrainedFP.push_back(OutChain);
8871 break;
8873 PendingConstrainedFPStrict.push_back(OutChain);
8874 break;
8875 }
8876}
8877
8878void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8879 const ConstrainedFPIntrinsic &FPI) {
8880 SDLoc sdl = getCurSDLoc();
8881
8882 // We do not need to serialize constrained FP intrinsics against
8883 // each other or against (nonvolatile) loads, so they can be
8884 // chained like loads.
8886 SDValue Chain = getFPOperationRoot(EB);
8888 Opers.push_back(Chain);
8889 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8890 Opers.push_back(getValue(FPI.getArgOperand(I)));
8891
8892 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8893 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
8894 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
8895
8896 SDNodeFlags Flags;
8898 Flags.setNoFPExcept(true);
8899
8900 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
8901 Flags.copyFMF(*FPOp);
8902
8903 unsigned Opcode;
8904 switch (FPI.getIntrinsicID()) {
8905 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8906#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8907 case Intrinsic::INTRINSIC: \
8908 Opcode = ISD::STRICT_##DAGN; \
8909 break;
8910#include "llvm/IR/ConstrainedOps.def"
8911 case Intrinsic::experimental_constrained_fmuladd: {
8912 Opcode = ISD::STRICT_FMA;
8913 // Break fmuladd into fmul and fadd.
8914 if (!TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
8915 Opers.pop_back();
8916 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8917 pushFPOpOutChain(Mul, EB);
8918 Opcode = ISD::STRICT_FADD;
8919 Opers.clear();
8920 Opers.push_back(Mul.getValue(1));
8921 Opers.push_back(Mul.getValue(0));
8922 Opers.push_back(getValue(FPI.getArgOperand(2)));
8923 }
8924 break;
8925 }
8926 }
8927
8928 // A few strict DAG nodes carry additional operands that are not
8929 // set up by the default code above.
8930 switch (Opcode) {
8931 default: break;
8933 Opers.push_back(
8934 DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
8935 break;
8936 case ISD::STRICT_FSETCC:
8937 case ISD::STRICT_FSETCCS: {
8938 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8939 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8940 if (DAG.isKnownNeverNaN(Opers[1]) && DAG.isKnownNeverNaN(Opers[2]))
8941 Condition = getFCmpCodeWithoutNaN(Condition);
8942 Opers.push_back(DAG.getCondCode(Condition));
8943 break;
8944 }
8945 }
8946
8947 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8948 pushFPOpOutChain(Result, EB);
8949
8950 SDValue FPResult = Result.getValue(0);
8951 setValue(&FPI, FPResult);
8952}
8953
8954static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8955 std::optional<unsigned> ResOPC;
8956 switch (VPIntrin.getIntrinsicID()) {
8957 case Intrinsic::vp_cttz_elts: {
8958 bool IsZeroPoison = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8959 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_POISON : ISD::VP_CTTZ_ELTS;
8960 break;
8961 }
8962#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8963 case Intrinsic::VPID: \
8964 ResOPC = ISD::VPSD; \
8965 break;
8966#include "llvm/IR/VPIntrinsics.def"
8967 }
8968
8969 if (!ResOPC)
8971 "Inconsistency: no SDNode available for this VPIntrinsic!");
8972
8973 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8974 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8975 if (VPIntrin.getFastMathFlags().allowReassoc())
8976 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8977 : ISD::VP_REDUCE_FMUL;
8978 }
8979
8980 return *ResOPC;
8981}
8982
8983void SelectionDAGBuilder::visitVPLoad(
8984 const VPIntrinsic &VPIntrin, EVT VT,
8985 const SmallVectorImpl<SDValue> &OpValues) {
8986 SDLoc DL = getCurSDLoc();
8987 Value *PtrOperand = VPIntrin.getArgOperand(0);
8988 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8989 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8990 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8991 SDValue LD;
8992 // Do not serialize variable-length loads of constant memory with
8993 // anything.
8994 if (!Alignment)
8995 Alignment = DAG.getEVTAlign(VT);
8996 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8997 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
8998 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8999 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9000 MachineMemOperand::Flags MMOFlags =
9002 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9003 MachinePointerInfo(PtrOperand), MMOFlags,
9005 MMOMetadata(AAInfo, Ranges));
9006 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9007 MMO, false /*IsExpanding */);
9008 if (AddToChain)
9009 PendingLoads.push_back(LD.getValue(1));
9010 setValue(&VPIntrin, LD);
9011}
9012
9013void SelectionDAGBuilder::visitVPLoadFF(
9014 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,
9015 const SmallVectorImpl<SDValue> &OpValues) {
9016 assert(OpValues.size() == 3 && "Unexpected number of operands");
9017 SDLoc DL = getCurSDLoc();
9018 Value *PtrOperand = VPIntrin.getArgOperand(0);
9019 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9020 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9021 const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
9022 SDValue LD;
9023 // Do not serialize variable-length loads of constant memory with
9024 // anything.
9025 if (!Alignment)
9026 Alignment = DAG.getEVTAlign(VT);
9027 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
9028 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
9029 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9030 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9031 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
9033 MMOMetadata(AAInfo, Ranges));
9034 LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9035 MMO);
9036 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));
9037 if (AddToChain)
9038 PendingLoads.push_back(LD.getValue(2));
9039 setValue(&VPIntrin, DAG.getMergeValues({LD.getValue(0), Trunc}, DL));
9040}
9041
9042void SelectionDAGBuilder::visitVPGather(
9043 const VPIntrinsic &VPIntrin, EVT VT,
9044 const SmallVectorImpl<SDValue> &OpValues) {
9045 SDLoc DL = getCurSDLoc();
9046 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9047 Value *PtrOperand = VPIntrin.getArgOperand(0);
9048 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9049 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9050 const MDNode *Ranges = getRangeMetadata(VPIntrin);
9051 SDValue LD;
9052 if (!Alignment)
9053 Alignment = DAG.getEVTAlign(VT.getScalarType());
9054 unsigned AS =
9055 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9056 MachineMemOperand::Flags MMOFlags =
9058 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9059 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9060 *Alignment, MMOMetadata(AAInfo, Ranges));
9061 SDValue Base, Index, Scale;
9062 bool UniformBase =
9063 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
9064 VT.getScalarStoreSize());
9065 if (!UniformBase) {
9066 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
9067 Index = getValue(PtrOperand);
9068 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
9069 }
9070 EVT IdxVT = Index.getValueType();
9071 EVT EltTy = IdxVT.getVectorElementType();
9072 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
9073 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
9074 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
9075 }
9076 LD = DAG.getGatherVP(
9077 DAG.getVTList(VT, MVT::Other), VT, DL,
9078 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
9080 PendingLoads.push_back(LD.getValue(1));
9081 setValue(&VPIntrin, LD);
9082}
9083
9084void SelectionDAGBuilder::visitVPStore(
9085 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9086 SDLoc DL = getCurSDLoc();
9087 Value *PtrOperand = VPIntrin.getArgOperand(1);
9088 EVT VT = OpValues[0].getValueType();
9089 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9090 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9091 SDValue ST;
9092 if (!Alignment)
9093 Alignment = DAG.getEVTAlign(VT);
9094 SDValue Ptr = OpValues[1];
9095 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
9096 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9097 MachineMemOperand::Flags MMOFlags =
9099 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9100 MachinePointerInfo(PtrOperand), MMOFlags,
9101 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
9102 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
9103 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
9104 /* IsTruncating */ false, /*IsCompressing*/ false);
9105 DAG.setRoot(ST);
9106 setValue(&VPIntrin, ST);
9107}
9108
9109void SelectionDAGBuilder::visitVPScatter(
9110 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9111 SDLoc DL = getCurSDLoc();
9112 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9113 Value *PtrOperand = VPIntrin.getArgOperand(1);
9114 EVT VT = OpValues[0].getValueType();
9115 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9116 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9117 SDValue ST;
9118 if (!Alignment)
9119 Alignment = DAG.getEVTAlign(VT.getScalarType());
9120 unsigned AS =
9121 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9122 MachineMemOperand::Flags MMOFlags =
9124 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9125 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9126 *Alignment, AAInfo);
9127 SDValue Base, Index, Scale;
9128 bool UniformBase =
9129 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
9130 VT.getScalarStoreSize());
9131 if (!UniformBase) {
9132 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
9133 Index = getValue(PtrOperand);
9134 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
9135 }
9136 EVT IdxVT = Index.getValueType();
9137 EVT EltTy = IdxVT.getVectorElementType();
9138 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
9139 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
9140 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
9141 }
9142 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
9143 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
9144 OpValues[2], OpValues[3]},
9145 MMO, ISD::SIGNED_SCALED);
9146 DAG.setRoot(ST);
9147 setValue(&VPIntrin, ST);
9148}
9149
9150void SelectionDAGBuilder::visitVPStridedLoad(
9151 const VPIntrinsic &VPIntrin, EVT VT,
9152 const SmallVectorImpl<SDValue> &OpValues) {
9153 SDLoc DL = getCurSDLoc();
9154 Value *PtrOperand = VPIntrin.getArgOperand(0);
9155 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9156 if (!Alignment)
9157 Alignment = DAG.getEVTAlign(VT.getScalarType());
9158 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9159 const MDNode *Ranges = getRangeMetadata(VPIntrin);
9160 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
9161 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
9162 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9163 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9164 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9165 MachineMemOperand::Flags MMOFlags =
9167 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9168 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9169 *Alignment, MMOMetadata(AAInfo, Ranges));
9170
9171 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
9172 OpValues[2], OpValues[3], MMO,
9173 false /*IsExpanding*/);
9174
9175 if (AddToChain)
9176 PendingLoads.push_back(LD.getValue(1));
9177 setValue(&VPIntrin, LD);
9178}
9179
9180void SelectionDAGBuilder::visitVPStridedStore(
9181 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9182 SDLoc DL = getCurSDLoc();
9183 Value *PtrOperand = VPIntrin.getArgOperand(1);
9184 EVT VT = OpValues[0].getValueType();
9185 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9186 if (!Alignment)
9187 Alignment = DAG.getEVTAlign(VT.getScalarType());
9188 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9189 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9190 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9191 MachineMemOperand::Flags MMOFlags =
9193 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9194 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9195 *Alignment, AAInfo);
9196
9197 SDValue ST = DAG.getStridedStoreVP(
9198 getMemoryRoot(), DL, OpValues[0], OpValues[1],
9199 DAG.getPOISON(OpValues[1].getValueType()), OpValues[2], OpValues[3],
9200 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
9201 /*IsCompressing*/ false);
9202
9203 DAG.setRoot(ST);
9204 setValue(&VPIntrin, ST);
9205}
9206
9207void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
9208 const VPIntrinsic &VPIntrin) {
9209 SDLoc DL = getCurSDLoc();
9210 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
9211
9212 auto IID = VPIntrin.getIntrinsicID();
9213
9214 SmallVector<EVT, 4> ValueVTs;
9215 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9216 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
9217 SDVTList VTs = DAG.getVTList(ValueVTs);
9218
9219 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
9220
9221 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
9222 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
9223 "Unexpected target EVL type");
9224
9225 // Request operands.
9226 SmallVector<SDValue, 7> OpValues;
9227 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
9228 auto Op = getValue(VPIntrin.getArgOperand(I));
9229 if (I == EVLParamPos)
9230 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
9231 OpValues.push_back(Op);
9232 }
9233
9234 switch (Opcode) {
9235 default: {
9236 SDNodeFlags SDFlags;
9237 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
9238 SDFlags.copyFMF(*FPMO);
9239 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
9240 setValue(&VPIntrin, Result);
9241 break;
9242 }
9243 case ISD::VP_LOAD:
9244 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
9245 break;
9246 case ISD::VP_LOAD_FF:
9247 visitVPLoadFF(VPIntrin, ValueVTs[0], ValueVTs[1], OpValues);
9248 break;
9249 case ISD::VP_GATHER:
9250 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
9251 break;
9252 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9253 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
9254 break;
9255 case ISD::VP_STORE:
9256 visitVPStore(VPIntrin, OpValues);
9257 break;
9258 case ISD::VP_SCATTER:
9259 visitVPScatter(VPIntrin, OpValues);
9260 break;
9261 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9262 visitVPStridedStore(VPIntrin, OpValues);
9263 break;
9264 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
9265 case ISD::VP_CTTZ_ELTS: {
9266 SDValue Result =
9267 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
9268 setValue(&VPIntrin, Result);
9269 break;
9270 }
9271 }
9272}
9273
9275 const BasicBlock *EHPadBB,
9276 MCSymbol *&BeginLabel) {
9277 MachineFunction &MF = DAG.getMachineFunction();
9278
9279 // Skip emitting EH_LABEL on targets whose exception tables don't reference
9280 // them (32-bit x86 SEH, Wasm).
9282 FuncInfo.ExceptionModel)) {
9283 BeginLabel = nullptr;
9284 return Chain;
9285 }
9286
9287 // Insert a label before the invoke call to mark the try range. This can be
9288 // used to detect deletion of the invoke via the MachineModuleInfo.
9289 BeginLabel = MF.getContext().createTempSymbol();
9290
9291 // For SjLj, keep track of which landing pads go with which invokes
9292 // so as to maintain the ordering of pads in the LSDA.
9293 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
9294 if (CallSiteIndex) {
9295 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
9296 LPadToCallSiteMap[FuncInfo.getMBB(EHPadBB)].push_back(CallSiteIndex);
9297
9298 // Now that the call site is handled, stop tracking it.
9299 FuncInfo.setCurrentCallSite(0);
9300 }
9301
9302 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
9303}
9304
9305SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
9306 const BasicBlock *EHPadBB,
9307 MCSymbol *BeginLabel) {
9308 // No labels were emitted.
9309 if (!BeginLabel)
9310 return Chain;
9311
9313
9314 // Insert a label at the end of the invoke call to mark the try range. This
9315 // can be used to detect deletion of the invoke via the MachineModuleInfo.
9316 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
9317 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
9318
9319 // Inform MachineModuleInfo of range.
9321 // There is a platform (e.g. wasm) that uses funclet style IR but does not
9322 // actually use outlined funclets and their LSDA info style.
9323 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
9324 assert(II && "II should've been set");
9325 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
9326 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
9327 } else if (!isScopedEHPersonality(Pers)) {
9328 assert(EHPadBB);
9329 MF.addInvoke(FuncInfo.getMBB(EHPadBB), BeginLabel, EndLabel);
9330 }
9331
9332 return Chain;
9333}
9334
9335std::pair<SDValue, SDValue>
9337 const BasicBlock *EHPadBB) {
9338 MCSymbol *BeginLabel = nullptr;
9339
9340 if (EHPadBB) {
9341 // Both PendingLoads and PendingExports must be flushed here;
9342 // this call might not return.
9343 (void)getRoot();
9344 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
9345 CLI.setChain(getRoot());
9346 }
9347
9348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9349 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
9350
9351 assert((CLI.IsTailCall || Result.second.getNode()) &&
9352 "Non-null chain expected with non-tail call!");
9353 assert((Result.second.getNode() || !Result.first.getNode()) &&
9354 "Null value expected with tail call!");
9355
9356 if (!Result.second.getNode()) {
9357 // As a special case, a null chain means that a tail call has been emitted
9358 // and the DAG root is already updated.
9359 HasTailCall = true;
9360
9361 // Since there's no actual continuation from this block, nothing can be
9362 // relying on us setting vregs for them.
9363 PendingExports.clear();
9364 } else {
9365 DAG.setRoot(Result.second);
9366 }
9367
9368 if (EHPadBB) {
9369 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
9370 BeginLabel));
9371 Result.second = getRoot();
9372 }
9373
9374 return Result;
9375}
9376
9378 bool isMustTailCall = CB.isMustTailCall();
9379
9380 // Avoid emitting tail calls in functions with the disable-tail-calls
9381 // attribute.
9382 const Function *Caller = CB.getParent()->getParent();
9383 if (!isMustTailCall &&
9384 Caller->getFnAttribute("disable-tail-calls").getValueAsBool())
9385 return false;
9386
9387 // We can't tail call inside a function with a swifterror argument. Lowering
9388 // does not support this yet. It would have to move into the swifterror
9389 // register before the call.
9390 if (DAG.hasSwiftErrorArg())
9391 return false;
9392
9393 // Check if target-independent constraints permit a tail call here.
9394 // Target-dependent constraints are checked within TLI->LowerCallTo.
9395 return isInTailCallPosition(CB, DAG.getTarget());
9396}
9397
9399 bool isTailCall, bool isMustTailCall,
9400 const BasicBlock *EHPadBB,
9401 const TargetLowering::PtrAuthInfo *PAI) {
9402 auto &DL = DAG.getDataLayout();
9403 FunctionType *FTy = CB.getFunctionType();
9404 Type *RetTy = CB.getType();
9405
9407 Args.reserve(CB.arg_size());
9408
9409 const Value *SwiftErrorVal = nullptr;
9410 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9411
9412 if (isTailCall)
9413 isTailCall = canTailCall(CB);
9414
9415 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
9416 const Value *V = *I;
9417
9418 // Skip empty types
9419 if (V->getType()->isEmptyTy())
9420 continue;
9421
9422 SDValue ArgNode = getValue(V);
9423 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
9424 Entry.setAttributes(&CB, I - CB.arg_begin());
9425
9426 // Use swifterror virtual register as input to the call.
9427 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
9428 SwiftErrorVal = V;
9429 // We find the virtual register for the actual swifterror argument.
9430 // Instead of using the Value, we use the virtual register instead.
9431 Entry.Node =
9432 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
9433 EVT(TLI.getPointerTy(DL)));
9434 }
9435
9436 Args.push_back(Entry);
9437
9438 // If we have an explicit sret argument that is an Instruction, (i.e., it
9439 // might point to function-local memory), we can't meaningfully tail-call.
9440 if (Entry.IsSRet && isa<Instruction>(V))
9441 isTailCall = false;
9442 }
9443
9444 // If call site has a cfguardtarget operand bundle, create and add an
9445 // additional ArgListEntry.
9446 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
9447 Value *V = Bundle->Inputs[0];
9449 Entry.IsCFGuardTarget = true;
9450 Args.push_back(Entry);
9451 }
9452
9453 // Disable tail calls if there is an swifterror argument. Targets have not
9454 // been updated to support tail calls.
9455 if (TLI.supportSwiftError() && SwiftErrorVal)
9456 isTailCall = false;
9457
9458 ConstantInt *CFIType = nullptr;
9459 if (CB.isIndirectCall()) {
9460 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
9461 if (!TLI.supportKCFIBundles())
9463 "Target doesn't support calls with kcfi operand bundles.");
9464 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
9465 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
9466 }
9467 }
9468
9469 SDValue ConvControlToken;
9470 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
9471 auto *Token = Bundle->Inputs[0].get();
9472 ConvControlToken = getValue(Token);
9473 }
9474
9475 GlobalValue *DeactivationSymbol = nullptr;
9477 DeactivationSymbol = cast<GlobalValue>(Bundle->Inputs[0].get());
9478 }
9479
9482 .setChain(getRoot())
9483 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
9484 .setTailCall(isTailCall)
9488 .setCFIType(CFIType)
9489 .setConvergenceControlToken(ConvControlToken)
9490 .setDeactivationSymbol(DeactivationSymbol);
9491
9492 // Set the pointer authentication info if we have it.
9493 if (PAI) {
9494 if (!TLI.supportPtrAuthBundles())
9496 "This target doesn't support calls with ptrauth operand bundles.");
9497 CLI.setPtrAuth(*PAI);
9498 }
9499
9500 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9501
9502 if (Result.first.getNode()) {
9503 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
9504 Result.first = lowerNoFPClassToAssertNoFPClass(DAG, CB, Result.first);
9505 setValue(&CB, Result.first);
9506 }
9507
9508 // The last element of CLI.InVals has the SDValue for swifterror return.
9509 // Here we copy it to a virtual register and update SwiftErrorMap for
9510 // book-keeping.
9511 if (SwiftErrorVal && TLI.supportSwiftError()) {
9512 // Get the last element of InVals.
9513 SDValue Src = CLI.InVals.back();
9514 Register VReg =
9515 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
9516 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
9517 DAG.setRoot(CopyNode);
9518 }
9519}
9520
9521static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
9522 SelectionDAGBuilder &Builder) {
9523 // Check to see if this load can be trivially constant folded, e.g. if the
9524 // input is from a string literal.
9525 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
9526 // Cast pointer to the type we really want to load.
9527 Type *LoadTy =
9528 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
9529 if (LoadVT.isVector())
9530 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
9531 if (const Constant *LoadCst =
9532 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
9533 LoadTy, Builder.DAG.getDataLayout()))
9534 return Builder.getValue(LoadCst);
9535 }
9536
9537 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9538 // still constant memory, the input chain can be the entry node.
9539 SDValue Root;
9540 bool ConstantMemory = false;
9541
9542 // Do not serialize (non-volatile) loads of constant memory with anything.
9543 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) {
9544 Root = Builder.DAG.getEntryNode();
9545 ConstantMemory = true;
9546 } else {
9547 // Do not serialize non-volatile loads against each other.
9548 Root = Builder.DAG.getRoot();
9549 }
9550
9551 SDValue Ptr = Builder.getValue(PtrVal);
9552 SDValue LoadVal =
9553 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
9554 MachinePointerInfo(PtrVal), Align(1));
9555
9556 if (!ConstantMemory)
9557 Builder.PendingLoads.push_back(LoadVal.getValue(1));
9558 return LoadVal;
9559}
9560
9561/// Record the value for an instruction that produces an integer result,
9562/// converting the type where necessary.
9563void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9564 SDValue Value,
9565 bool IsSigned) {
9566 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
9567 I.getType(), true);
9568 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
9569 setValue(&I, Value);
9570}
9571
9572/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9573/// true and lower it. Otherwise return false, and it will be lowered like a
9574/// normal call.
9575/// The caller already checked that \p I calls the appropriate LibFunc with a
9576/// correct prototype.
9577bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9578 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
9579 const Value *Size = I.getArgOperand(2);
9580 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
9581 if (CSize && CSize->getZExtValue() == 0) {
9582 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
9583 I.getType(), true);
9584 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
9585 return true;
9586 }
9587
9588 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9589 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9590 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
9591 getValue(Size), &I);
9592 if (Res.first.getNode()) {
9593 processIntegerCallValue(I, Res.first, true);
9594 PendingLoads.push_back(Res.second);
9595 return true;
9596 }
9597
9598 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9599 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9600 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
9601 return false;
9602
9603 // If the target has a fast compare for the given size, it will return a
9604 // preferred load type for that size. Require that the load VT is legal and
9605 // that the target supports unaligned loads of that type. Otherwise, return
9606 // INVALID.
9607 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9608 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9609 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9610 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9611 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9612 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9613 // TODO: Check alignment of src and dest ptrs.
9614 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9615 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9616 if (!TLI.isTypeLegal(LVT) ||
9617 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
9620 }
9621
9622 return LVT;
9623 };
9624
9625 // This turns into unaligned loads. We only do this if the target natively
9626 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9627 // we'll only produce a small number of byte loads.
9628 MVT LoadVT;
9629 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9630 switch (NumBitsToCompare) {
9631 default:
9632 return false;
9633 case 16:
9634 LoadVT = MVT::i16;
9635 break;
9636 case 32:
9637 LoadVT = MVT::i32;
9638 break;
9639 case 64:
9640 case 128:
9641 case 256:
9642 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9643 break;
9644 }
9645
9646 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9647 return false;
9648
9649 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
9650 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
9651
9652 // Bitcast to a wide integer type if the loads are vectors.
9653 if (LoadVT.isVector()) {
9654 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
9655 LoadL = DAG.getBitcast(CmpVT, LoadL);
9656 LoadR = DAG.getBitcast(CmpVT, LoadR);
9657 }
9658
9659 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
9660 processIntegerCallValue(I, Cmp, false);
9661 return true;
9662}
9663
9664/// See if we can lower a memchr call into an optimized form. If so, return
9665/// true and lower it. Otherwise return false, and it will be lowered like a
9666/// normal call.
9667/// The caller already checked that \p I calls the appropriate LibFunc with a
9668/// correct prototype.
9669bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9670 const Value *Src = I.getArgOperand(0);
9671 const Value *Char = I.getArgOperand(1);
9672 const Value *Length = I.getArgOperand(2);
9673
9674 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9675 std::pair<SDValue, SDValue> Res =
9676 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
9677 getValue(Src), getValue(Char), getValue(Length),
9678 MachinePointerInfo(Src));
9679 if (Res.first.getNode()) {
9680 setValue(&I, Res.first);
9681 PendingLoads.push_back(Res.second);
9682 return true;
9683 }
9684
9685 return false;
9686}
9687
9688/// See if we can lower a memccpy call into an optimized form. If so, return
9689/// true and lower it, otherwise return false and it will be lowered like a
9690/// normal call.
9691/// The caller already checked that \p I calls the appropriate LibFunc with a
9692/// correct prototype.
9693bool SelectionDAGBuilder::visitMemCCpyCall(const CallInst &I) {
9694 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9695 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemccpy(
9696 DAG, getCurSDLoc(), DAG.getRoot(), getValue(I.getArgOperand(0)),
9697 getValue(I.getArgOperand(1)), getValue(I.getArgOperand(2)),
9698 getValue(I.getArgOperand(3)), &I);
9699
9700 if (Res.first) {
9701 processIntegerCallValue(I, Res.first, true);
9702 PendingLoads.push_back(Res.second);
9703 return true;
9704 }
9705 return false;
9706}
9707
9708/// See if we can lower a mempcpy call into an optimized form. If so, return
9709/// true and lower it. Otherwise return false, and it will be lowered like a
9710/// normal call.
9711/// The caller already checked that \p I calls the appropriate LibFunc with a
9712/// correct prototype.
9713bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9714 SDValue Dst = getValue(I.getArgOperand(0));
9715 SDValue Src = getValue(I.getArgOperand(1));
9716 SDValue Size = getValue(I.getArgOperand(2));
9717
9718 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
9719 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
9720
9721 SDLoc sdl = getCurSDLoc();
9722
9723 // In the mempcpy context we need to pass in a false value for isTailCall
9724 // because the return pointer needs to be adjusted by the size of
9725 // the copied memory.
9726 SDValue Root = getMemoryRoot();
9727 SDValue MC = DAG.getMemcpy(
9728 Root, sdl, Dst, Src, Size, DstAlign, SrcAlign, false, false,
9729 /*CI=*/nullptr, std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
9730 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
9731 assert(MC.getNode() != nullptr &&
9732 "** memcpy should not be lowered as TailCall in mempcpy context **");
9733 DAG.setRoot(MC);
9734
9735 // Check if Size needs to be truncated or extended.
9736 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
9737
9738 // Adjust return pointer to point just past the last dst byte.
9739 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Dst, Size, sdl);
9740 setValue(&I, DstPlusSize);
9741 return true;
9742}
9743
9744/// See if we can lower a strcpy call into an optimized form. If so, return
9745/// true and lower it, otherwise return false and it will be lowered like a
9746/// normal call.
9747/// The caller already checked that \p I calls the appropriate LibFunc with a
9748/// correct prototype.
9749bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9750 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9751
9752 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9753 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcpy(
9754 DAG, getCurSDLoc(), getRoot(), getValue(Arg0), getValue(Arg1),
9755 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), isStpcpy, &I);
9756 if (Res.first.getNode()) {
9757 setValue(&I, Res.first);
9758 DAG.setRoot(Res.second);
9759 return true;
9760 }
9761
9762 return false;
9763}
9764
9765/// See if we can lower a strcmp call into an optimized form. If so, return
9766/// true and lower it, otherwise return false and it will be lowered like a
9767/// normal call.
9768/// The caller already checked that \p I calls the appropriate LibFunc with a
9769/// correct prototype.
9770bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9771 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9772
9773 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9774 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcmp(
9775 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), getValue(Arg1),
9776 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), &I);
9777 if (Res.first.getNode()) {
9778 processIntegerCallValue(I, Res.first, true);
9779 PendingLoads.push_back(Res.second);
9780 return true;
9781 }
9782
9783 return false;
9784}
9785
9786/// See if we can lower a strlen call into an optimized form. If so, return
9787/// true and lower it, otherwise return false and it will be lowered like a
9788/// normal call.
9789/// The caller already checked that \p I calls the appropriate LibFunc with a
9790/// correct prototype.
9791bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9792 const Value *Arg0 = I.getArgOperand(0);
9793
9794 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9795 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(
9796 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), &I);
9797 if (Res.first.getNode()) {
9798 processIntegerCallValue(I, Res.first, false);
9799 PendingLoads.push_back(Res.second);
9800 return true;
9801 }
9802
9803 return false;
9804}
9805
9806/// See if we can lower a strnlen call into an optimized form. If so, return
9807/// true and lower it, otherwise return false and it will be lowered like a
9808/// normal call.
9809/// The caller already checked that \p I calls the appropriate LibFunc with a
9810/// correct prototype.
9811bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9812 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9813
9814 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9815 std::pair<SDValue, SDValue> Res =
9816 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
9817 getValue(Arg0), getValue(Arg1),
9818 MachinePointerInfo(Arg0));
9819 if (Res.first.getNode()) {
9820 processIntegerCallValue(I, Res.first, false);
9821 PendingLoads.push_back(Res.second);
9822 return true;
9823 }
9824
9825 return false;
9826}
9827
9828/// See if we can lower a Strstr call into an optimized form. If so, return
9829/// true and lower it, otherwise return false and it will be lowered like a
9830/// normal call.
9831/// The caller already checked that \p I calls the appropriate LibFunc with a
9832/// correct prototype.
9833bool SelectionDAGBuilder::visitStrstrCall(const CallInst &I) {
9834 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9835 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9836 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrstr(
9837 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), getValue(Arg1), &I);
9838 if (Res.first) {
9839 processIntegerCallValue(I, Res.first, false);
9840 PendingLoads.push_back(Res.second);
9841 return true;
9842 }
9843 return false;
9844}
9845
9846/// See if we can lower a unary floating-point operation into an SDNode with
9847/// the specified Opcode. If so, return true and lower it, otherwise return
9848/// false and it will be lowered like a normal call.
9849/// The caller already checked that \p I calls the appropriate LibFunc with a
9850/// correct prototype.
9851bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9852 unsigned Opcode) {
9853 // We already checked this call's prototype; verify it doesn't modify errno.
9854 // Do not perform optimizations for call sites that require strict
9855 // floating-point semantics.
9856 if (!I.onlyReadsMemory() || I.isStrictFP())
9857 return false;
9858
9859 SDNodeFlags Flags;
9860 Flags.copyFMF(cast<FPMathOperator>(I));
9861
9862 SDValue Tmp = getValue(I.getArgOperand(0));
9863 setValue(&I,
9864 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
9865 return true;
9866}
9867
9868/// See if we can lower a binary floating-point operation into an SDNode with
9869/// the specified Opcode. If so, return true and lower it. Otherwise return
9870/// false, and it will be lowered like a normal call.
9871/// The caller already checked that \p I calls the appropriate LibFunc with a
9872/// correct prototype.
9873bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9874 unsigned Opcode) {
9875 // We already checked this call's prototype; verify it doesn't modify errno.
9876 // Do not perform optimizations for call sites that require strict
9877 // floating-point semantics.
9878 if (!I.onlyReadsMemory() || I.isStrictFP())
9879 return false;
9880
9881 SDNodeFlags Flags;
9882 Flags.copyFMF(cast<FPMathOperator>(I));
9883
9884 SDValue Tmp0 = getValue(I.getArgOperand(0));
9885 SDValue Tmp1 = getValue(I.getArgOperand(1));
9886 EVT VT = Tmp0.getValueType();
9887 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
9888 return true;
9889}
9890
9891void SelectionDAGBuilder::visitCall(const CallInst &I) {
9892 // Handle inline assembly differently.
9893 if (I.isInlineAsm()) {
9894 visitInlineAsm(I);
9895 return;
9896 }
9897
9899
9900 if (Function *F = I.getCalledFunction()) {
9901 if (F->isDeclaration()) {
9902 // Is this an LLVM intrinsic?
9903 if (unsigned IID = F->getIntrinsicID()) {
9904 visitIntrinsicCall(I, IID);
9905 return;
9906 }
9907 }
9908
9909 // Check for well-known libc/libm calls. If the function is internal, it
9910 // can't be a library call. Don't do the check if marked as nobuiltin for
9911 // some reason.
9912 // This code should not handle libcalls that are already canonicalized to
9913 // intrinsics by the middle-end.
9914 LibFunc Func = !I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName()
9915 ? LibInfo->getLibFunc(*F)
9916 : NotLibFunc;
9917 if (LibInfo->hasOptimizedCodeGen(Func)) {
9918 switch (Func) {
9919 default: break;
9920 case LibFunc_bcmp:
9921 if (visitMemCmpBCmpCall(I))
9922 return;
9923 break;
9924 case LibFunc_sin:
9925 case LibFunc_sinf:
9926 case LibFunc_sinl:
9927 if (visitUnaryFloatCall(I, ISD::FSIN))
9928 return;
9929 break;
9930 case LibFunc_cos:
9931 case LibFunc_cosf:
9932 case LibFunc_cosl:
9933 if (visitUnaryFloatCall(I, ISD::FCOS))
9934 return;
9935 break;
9936 case LibFunc_tan:
9937 case LibFunc_tanf:
9938 case LibFunc_tanl:
9939 if (visitUnaryFloatCall(I, ISD::FTAN))
9940 return;
9941 break;
9942 case LibFunc_asin:
9943 case LibFunc_asinf:
9944 case LibFunc_asinl:
9945 if (visitUnaryFloatCall(I, ISD::FASIN))
9946 return;
9947 break;
9948 case LibFunc_acos:
9949 case LibFunc_acosf:
9950 case LibFunc_acosl:
9951 if (visitUnaryFloatCall(I, ISD::FACOS))
9952 return;
9953 break;
9954 case LibFunc_atan:
9955 case LibFunc_atanf:
9956 case LibFunc_atanl:
9957 if (visitUnaryFloatCall(I, ISD::FATAN))
9958 return;
9959 break;
9960 case LibFunc_atan2:
9961 case LibFunc_atan2f:
9962 case LibFunc_atan2l:
9963 if (visitBinaryFloatCall(I, ISD::FATAN2))
9964 return;
9965 break;
9966 case LibFunc_sinh:
9967 case LibFunc_sinhf:
9968 case LibFunc_sinhl:
9969 if (visitUnaryFloatCall(I, ISD::FSINH))
9970 return;
9971 break;
9972 case LibFunc_cosh:
9973 case LibFunc_coshf:
9974 case LibFunc_coshl:
9975 if (visitUnaryFloatCall(I, ISD::FCOSH))
9976 return;
9977 break;
9978 case LibFunc_tanh:
9979 case LibFunc_tanhf:
9980 case LibFunc_tanhl:
9981 if (visitUnaryFloatCall(I, ISD::FTANH))
9982 return;
9983 break;
9984 case LibFunc_sqrt:
9985 case LibFunc_sqrtf:
9986 case LibFunc_sqrtl:
9987 case LibFunc_sqrt_finite:
9988 case LibFunc_sqrtf_finite:
9989 case LibFunc_sqrtl_finite:
9990 if (visitUnaryFloatCall(I, ISD::FSQRT))
9991 return;
9992 break;
9993 case LibFunc_log2:
9994 case LibFunc_log2f:
9995 case LibFunc_log2l:
9996 if (visitUnaryFloatCall(I, ISD::FLOG2))
9997 return;
9998 break;
9999 case LibFunc_exp2:
10000 case LibFunc_exp2f:
10001 case LibFunc_exp2l:
10002 if (visitUnaryFloatCall(I, ISD::FEXP2))
10003 return;
10004 break;
10005 case LibFunc_exp10:
10006 case LibFunc_exp10f:
10007 case LibFunc_exp10l:
10008 if (visitUnaryFloatCall(I, ISD::FEXP10))
10009 return;
10010 break;
10011 case LibFunc_strstr:
10012 if (visitStrstrCall(I))
10013 return;
10014 break;
10015 case LibFunc_memcmp:
10016 if (visitMemCmpBCmpCall(I))
10017 return;
10018 break;
10019 case LibFunc_memccpy:
10020 if (visitMemCCpyCall(I))
10021 return;
10022 break;
10023 case LibFunc_mempcpy:
10024 if (visitMemPCpyCall(I))
10025 return;
10026 break;
10027 case LibFunc_memchr:
10028 if (visitMemChrCall(I))
10029 return;
10030 break;
10031 case LibFunc_strcpy:
10032 if (visitStrCpyCall(I, false))
10033 return;
10034 break;
10035 case LibFunc_stpcpy:
10036 if (visitStrCpyCall(I, true))
10037 return;
10038 break;
10039 case LibFunc_strcmp:
10040 if (visitStrCmpCall(I))
10041 return;
10042 break;
10043 case LibFunc_strlen:
10044 if (visitStrLenCall(I))
10045 return;
10046 break;
10047 case LibFunc_strnlen:
10048 if (visitStrNLenCall(I))
10049 return;
10050 break;
10051 }
10052 }
10053 }
10054
10055 if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
10056 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
10057 return;
10058 }
10059
10060 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
10061 // have to do anything here to lower funclet bundles.
10062 // CFGuardTarget bundles are lowered in LowerCallTo.
10064 I, "calls",
10069
10070 SDValue Callee = getValue(I.getCalledOperand());
10071
10072 if (I.hasDeoptState())
10073 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
10074 else
10075 // Check if we can potentially perform a tail call. More detailed checking
10076 // is be done within LowerCallTo, after more information about the call is
10077 // known.
10078 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
10079}
10080
10082 const CallBase &CB, const BasicBlock *EHPadBB) {
10083 auto PAB = CB.getOperandBundle("ptrauth");
10084 const Value *CalleeV = CB.getCalledOperand();
10085
10086 // Gather the call ptrauth data from the operand bundle:
10087 // [ i32 <key>, i64 <discriminator> ]
10088 const auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
10089 const Value *Discriminator = PAB->Inputs[1];
10090
10091 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
10092 assert(Discriminator->getType()->isIntegerTy(64) &&
10093 "Invalid ptrauth discriminator");
10094
10095 // Look through ptrauth constants to find the raw callee.
10096 // Do a direct unauthenticated call if we found it and everything matches.
10097 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CalleeV))
10098 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
10099 DAG.getDataLayout()))
10100 return LowerCallTo(CB, getValue(CalleeCPA->getPointer()), CB.isTailCall(),
10101 CB.isMustTailCall(), EHPadBB);
10102
10103 // Functions should never be ptrauth-called directly.
10104 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
10105
10106 // Otherwise, do an authenticated indirect call.
10107 TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
10108 getValue(Discriminator)};
10109
10110 LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
10111 EHPadBB, &PAI);
10112}
10113
10114namespace {
10115
10116/// AsmOperandInfo - This contains information for each constraint that we are
10117/// lowering.
10118class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
10119public:
10120 /// CallOperand - If this is the result output operand or a clobber
10121 /// this is null, otherwise it is the incoming operand to the CallInst.
10122 /// This gets modified as the asm is processed.
10123 SDValue CallOperand;
10124
10125 /// AssignedRegs - If this is a register or register class operand, this
10126 /// contains the set of register corresponding to the operand.
10127 RegsForValue AssignedRegs;
10128
10129 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
10130 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
10131 }
10132
10133 /// Whether or not this operand accesses memory
10134 bool hasMemory(const TargetLowering &TLI) const {
10135 // Indirect operand accesses access memory.
10136 if (isIndirect)
10137 return true;
10138
10139 for (const auto &Code : Codes)
10140 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
10141 return true;
10142
10143 return false;
10144 }
10145};
10146
10147
10148} // end anonymous namespace
10149
10150/// Make sure that the output operand \p OpInfo and its corresponding input
10151/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
10152/// out).
10153static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
10154 SDISelAsmOperandInfo &MatchingOpInfo,
10155 SelectionDAG &DAG) {
10156 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
10157 return;
10158
10160 const auto &TLI = DAG.getTargetLoweringInfo();
10161
10162 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
10163 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
10164 OpInfo.ConstraintVT);
10165 std::pair<unsigned, const TargetRegisterClass *> InputRC =
10166 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
10167 MatchingOpInfo.ConstraintVT);
10168 const bool OutOpIsIntOrFP =
10169 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
10170 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
10171 MatchingOpInfo.ConstraintVT.isFloatingPoint();
10172 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
10173 // FIXME: error out in a more elegant fashion
10174 report_fatal_error("Unsupported asm: input constraint"
10175 " with a matching output constraint of"
10176 " incompatible type!");
10177 }
10178 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
10179}
10180
10181/// Get a direct memory input to behave well as an indirect operand.
10182/// This may introduce stores, hence the need for a \p Chain.
10183/// \return The (possibly updated) chain.
10184static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
10185 SDISelAsmOperandInfo &OpInfo,
10186 SelectionDAG &DAG) {
10188
10189 // If we don't have an indirect input, put it in the constpool if we can,
10190 // otherwise spill it to a stack slot.
10191 // TODO: This isn't quite right. We need to handle these according to
10192 // the addressing mode that the constraint wants. Also, this may take
10193 // an additional register for the computation and we don't want that
10194 // either.
10195
10196 // If the operand is a float, integer, or vector constant, spill to a
10197 // constant pool entry to get its address.
10198 const Value *OpVal = OpInfo.CallOperandVal;
10199 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
10201 OpInfo.CallOperand = DAG.getConstantPool(
10202 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
10203 return Chain;
10204 }
10205
10206 // Otherwise, create a stack slot and emit a store to it before the asm.
10207 Type *Ty = OpVal->getType();
10208 auto &DL = DAG.getDataLayout();
10209 TypeSize TySize = DL.getTypeAllocSize(Ty);
10212 int StackID = 0;
10213 if (TySize.isScalable())
10214 StackID = TFI->getStackIDForScalableVectors();
10215 int SSFI = MF.getFrameInfo().CreateStackObject(TySize.getKnownMinValue(),
10216 DL.getPrefTypeAlign(Ty), false,
10217 nullptr, StackID);
10218 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
10219 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
10221 TLI.getMemValueType(DL, Ty));
10222 OpInfo.CallOperand = StackSlot;
10223
10224 return Chain;
10225}
10226
10227/// GetRegistersForValue - Assign registers (virtual or physical) for the
10228/// specified operand. We prefer to assign virtual registers, to allow the
10229/// register allocator to handle the assignment process. However, if the asm
10230/// uses features that we can't model on machineinstrs, we have SDISel do the
10231/// allocation. This produces generally horrible, but correct, code.
10232///
10233/// OpInfo describes the operand
10234/// RefOpInfo describes the matching operand if any, the operand otherwise
10235static std::optional<unsigned>
10237 SDISelAsmOperandInfo &OpInfo,
10238 SDISelAsmOperandInfo &RefOpInfo) {
10239 LLVMContext &Context = *DAG.getContext();
10241
10245
10246 // No work to do for memory/address operands.
10247 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10248 OpInfo.ConstraintType == TargetLowering::C_Address)
10249 return std::nullopt;
10250
10251 // If this is a constraint for a single physreg, or a constraint for a
10252 // register class, find it.
10253 unsigned AssignedReg;
10254 const TargetRegisterClass *RC;
10255 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
10256 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
10257 // RC is unset only on failure. Return immediately.
10258 if (!RC)
10259 return std::nullopt;
10260
10261 // Get the actual register value type. This is important, because the user
10262 // may have asked for (e.g.) the AX register in i32 type. We need to
10263 // remember that AX is actually i16 to get the right extension.
10264 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
10265
10266 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10267 // If this is an FP operand in an integer register (or visa versa), or more
10268 // generally if the operand value disagrees with the register class we plan
10269 // to stick it in, fix the operand type.
10270 //
10271 // If this is an input value, the bitcast to the new type is done now.
10272 // Bitcast for output value is done at the end of visitInlineAsm().
10273 if ((OpInfo.Type == InlineAsm::isOutput ||
10274 OpInfo.Type == InlineAsm::isInput) &&
10275 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
10276 // Try to convert to the first EVT that the reg class contains. If the
10277 // types are identical size, use a bitcast to convert (e.g. two differing
10278 // vector types). Note: output bitcast is done at the end of
10279 // visitInlineAsm().
10280 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10281 // Exclude indirect inputs while they are unsupported because the code
10282 // to perform the load is missing and thus OpInfo.CallOperand still
10283 // refers to the input address rather than the pointed-to value.
10284 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
10285 OpInfo.CallOperand =
10286 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
10287 OpInfo.ConstraintVT = RegVT;
10288 // If the operand is an FP value and we want it in integer registers,
10289 // use the corresponding integer type. This turns an f64 value into
10290 // i64, which can be passed with two i32 values on a 32-bit machine.
10291 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10292 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
10293 // No integer type of that size to bitcast to (e.g. f80).
10294 if (!VT.isValid())
10295 return std::nullopt;
10296 if (OpInfo.Type == InlineAsm::isInput)
10297 OpInfo.CallOperand =
10298 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
10299 OpInfo.ConstraintVT = VT;
10300 }
10301 }
10302 }
10303
10304 // No need to allocate a matching input constraint since the constraint it's
10305 // matching to has already been allocated.
10306 if (OpInfo.isMatchingInputConstraint())
10307 return std::nullopt;
10308
10309 EVT ValueVT = OpInfo.ConstraintVT;
10310 if (OpInfo.ConstraintVT == MVT::Other)
10311 ValueVT = RegVT;
10312
10313 // Initialize NumRegs.
10314 unsigned NumRegs = 1;
10315 if (OpInfo.ConstraintVT != MVT::Other)
10316 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
10317
10318 // If this is a constraint for a specific physical register, like {r17},
10319 // assign it now.
10320
10321 // If this associated to a specific register, initialize iterator to correct
10322 // place. If virtual, make sure we have enough registers
10323
10324 // Initialize iterator if necessary
10327
10328 // Do not check for single registers.
10329 if (AssignedReg) {
10330 I = std::find(I, RC->end(), AssignedReg);
10331 if (I == RC->end()) {
10332 // RC does not contain the selected register, which indicates a
10333 // mismatch between the register and the required type/bitwidth.
10334 return {AssignedReg};
10335 }
10336 }
10337
10338 for (; NumRegs; --NumRegs, ++I) {
10339 assert(I != RC->end() && "Ran out of registers to allocate!");
10340 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
10341 Regs.push_back(R);
10342 }
10343
10344 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
10345 return std::nullopt;
10346}
10347
10348static unsigned
10350 const std::vector<SDValue> &AsmNodeOperands) {
10351 // Scan until we find the definition we already emitted of this operand.
10352 unsigned CurOp = InlineAsm::Op_FirstOperand;
10353 for (; OperandNo; --OperandNo) {
10354 // Advance to the next operand.
10355 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10356 const InlineAsm::Flag F(OpFlag);
10357 assert(
10358 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
10359 "Skipped past definitions?");
10360 CurOp += F.getNumOperandRegisters() + 1;
10361 }
10362 return CurOp;
10363}
10364
10365namespace {
10366
10367class ExtraFlags {
10368 unsigned Flags = 0;
10369
10370public:
10371 explicit ExtraFlags(const CallBase &Call) {
10372 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
10373 if (IA->hasSideEffects())
10375 if (IA->isAlignStack())
10377 if (IA->canThrow())
10379 if (Call.isConvergent())
10381 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
10382 }
10383
10384 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
10385 // Ideally, we would only check against memory constraints. However, the
10386 // meaning of an Other constraint can be target-specific and we can't easily
10387 // reason about it. Therefore, be conservative and set MayLoad/MayStore
10388 // for Other constraints as well.
10391 if (OpInfo.Type == InlineAsm::isInput)
10393 else if (OpInfo.Type == InlineAsm::isOutput)
10395 else if (OpInfo.Type == InlineAsm::isClobber)
10397 }
10398 }
10399
10400 unsigned get() const { return Flags; }
10401};
10402
10403} // end anonymous namespace
10404
10405static bool isFunction(SDValue Op) {
10406 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
10407 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10408 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
10409
10410 // In normal "call dllimport func" instruction (non-inlineasm) it force
10411 // indirect access by specifing call opcode. And usually specially print
10412 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
10413 // not do in this way now. (In fact, this is similar with "Data Access"
10414 // action). So here we ignore dllimport function.
10415 if (Fn && !Fn->hasDLLImportStorageClass())
10416 return true;
10417 }
10418 }
10419 return false;
10420}
10421
10422namespace {
10423
10424struct ConstraintDecisionInfo {
10425 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
10426 std::vector<SDValue> AsmNodeOperands;
10427 SDValue Glue, Chain;
10428 bool HasSideEffect = false;
10429 MCSymbol *BeginLabel = nullptr;
10430
10431 SmallVector<char> Buffer;
10432 raw_svector_ostream ErrorMsg;
10433
10434 ConstraintDecisionInfo() : ErrorMsg(Buffer) {}
10435};
10436
10437} // end anonymous namespace
10438
10439/// Construct operand info objects.
10440static bool
10441constructOperandInfo(ConstraintDecisionInfo &Info,
10442 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10443 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10444 ExtraFlags &ExtraInfo) {
10445 for (auto &T : TargetConstraints) {
10446 Info.ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
10447 SDISelAsmOperandInfo &OpInfo = Info.ConstraintOperands.back();
10448
10449 if (OpInfo.CallOperandVal)
10450 OpInfo.CallOperand = Builder.getValue(OpInfo.CallOperandVal);
10451
10452 if (!Info.HasSideEffect)
10453 Info.HasSideEffect = OpInfo.hasMemory(TLI);
10454
10455 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
10456 // FIXME: Could we compute this on OpInfo rather than T?
10457
10458 // Compute the constraint code and ConstraintType to use.
10459 TLI.ComputeConstraintToUse(T, SDValue());
10460
10461 if (T.ConstraintType == TargetLowering::C_Immediate && OpInfo.CallOperand &&
10462 !isa<ConstantSDNode>(OpInfo.CallOperand)) {
10463 // We've delayed emitting a diagnostic like the "n" constraint because
10464 // inlining could cause an integer showing up.
10465 Info.ErrorMsg << "constraint '" << T.ConstraintCode
10466 << "' expects an integer constant expression";
10467 return true;
10468 }
10469
10470 ExtraInfo.update(T);
10471 }
10472
10473 return false;
10474}
10475
10476/// Compute which constraint option to use for each operand.
10477static void
10478computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
10479 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10480 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10481 const TargetMachine &TM, SelectionDAG &DAG) {
10482 const auto *IA = cast<InlineAsm>(Call.getCalledOperand());
10484 IA->collectAsmStrs(AsmStrs);
10485
10486 int OpNo = -1;
10487 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10488 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10489 OpNo++;
10490
10491 // If this is an output operand with a matching input operand, look up the
10492 // matching input. If their types mismatch, e.g. one is an integer, the
10493 // other is floating point, or their sizes are different, flag it as an
10494 // error.
10495 if (OpInfo.hasMatchingInput()) {
10496 SDISelAsmOperandInfo &Input =
10497 Info.ConstraintOperands[OpInfo.MatchingInput];
10498 patchMatchingInput(OpInfo, Input, DAG);
10499 }
10500
10501 // Compute the constraint code and ConstraintType to use.
10502 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
10503
10504 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10505 OpInfo.Type == InlineAsm::isClobber) ||
10506 OpInfo.ConstraintType == TargetLowering::C_Address)
10507 continue;
10508
10509 // In Linux PIC model, there are 4 cases about value/label addressing:
10510 //
10511 // 1: Function call or Label jmp inside the module.
10512 // 2: Data access (such as global variable, static variable) inside module.
10513 // 3: Function call or Label jmp outside the module.
10514 // 4: Data access (such as global variable) outside the module.
10515 //
10516 // Due to current llvm inline asm architecture designed to not "recognize"
10517 // the asm code, there are quite troubles for us to treat mem addressing
10518 // differently for same value/adress used in different instructions.
10519 // For example, in pic model, call a func may in plt way or direclty
10520 // pc-related, but lea/mov a function adress may use got.
10521 //
10522 // Here we try to "recognize" function call for the case 1 and case 3 in
10523 // inline asm. And try to adjust the constraint for them.
10524 //
10525 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10526 // label, so here we don't handle jmp function label now, but we need to
10527 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10528 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
10529 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10531 OpInfo.isIndirect = false;
10532 OpInfo.ConstraintType = TargetLowering::C_Address;
10533 }
10534
10535 // If this is a memory input, and if the operand is not indirect, do what we
10536 // need to provide an address for the memory input.
10537 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10538 !OpInfo.isIndirect) {
10539 assert((OpInfo.isMultipleAlternative ||
10540 (OpInfo.Type == InlineAsm::isInput)) &&
10541 "Can only indirectify direct input operands!");
10542
10543 // Memory operands really want the address of the value.
10544 Info.Chain = getAddressForMemoryInput(Info.Chain, Builder.getCurSDLoc(),
10545 OpInfo, DAG);
10546
10547 // There is no longer a Value* corresponding to this operand.
10548 OpInfo.CallOperandVal = nullptr;
10549
10550 // It is now an indirect operand.
10551 OpInfo.isIndirect = true;
10552 }
10553 }
10554}
10555
10556/// Prepare DAG-level operands. As part of this, assign virtual and physical
10557/// registers for inputs and output.
10558static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info,
10559 const CallBase &Call,
10560 SelectionDAGBuilder &Builder,
10561 const TargetLowering &TLI,
10562 SelectionDAG &DAG) {
10563 SDLoc DL = Builder.getCurSDLoc();
10564 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10565 // Assign Registers.
10566 SDISelAsmOperandInfo &RefOpInfo =
10567 OpInfo.isMatchingInputConstraint()
10568 ? Info.ConstraintOperands[OpInfo.getMatchedOperand()]
10569 : OpInfo;
10570 const auto RegError = getRegistersForValue(DAG, DL, OpInfo, RefOpInfo);
10571 if (RegError) {
10572 const MachineFunction &MF = DAG.getMachineFunction();
10574 const char *RegName = TRI.getName(*RegError);
10575 Info.ErrorMsg << "register '" << RegName << "' allocated for constraint '"
10576 << OpInfo.ConstraintCode
10577 << "' does not match required type";
10578 return true;
10579 }
10580
10581 auto DetectWriteToReservedRegister = [&]() {
10582 const MachineFunction &MF = DAG.getMachineFunction();
10584
10585 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10586 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
10587 Info.ErrorMsg << "write to reserved register '"
10588 << TRI.getRegAsmName(Reg) << "'";
10589 return true;
10590 }
10591 }
10592
10593 return false;
10594 };
10595 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10596 (OpInfo.Type == InlineAsm::isInput &&
10597 !OpInfo.isMatchingInputConstraint())) &&
10598 "Only address as input operand is allowed.");
10599
10600 switch (OpInfo.Type) {
10602 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10603 const InlineAsm::ConstraintCode ConstraintID =
10604 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10606 "Failed to convert memory constraint code to constraint id.");
10607
10608 // Add information to the INLINEASM node to know about this output.
10610 OpFlags.setMemConstraint(ConstraintID);
10611 Info.AsmNodeOperands.push_back(
10612 DAG.getTargetConstant(OpFlags, DL, MVT::i32));
10613 Info.AsmNodeOperands.push_back(OpInfo.CallOperand);
10614 } else {
10615 // Otherwise, this outputs to a register (directly for C_Register /
10616 // C_RegisterClass, and a target-defined fashion for
10617 // C_Immediate/C_Other). Find a register that we can use.
10618 if (OpInfo.AssignedRegs.Regs.empty()) {
10619 Info.ErrorMsg << "could not allocate output register for "
10620 << "constraint '" << OpInfo.ConstraintCode << "'";
10621 return true;
10622 }
10623
10624 if (DetectWriteToReservedRegister())
10625 return true;
10626
10627 // Add information to the INLINEASM node to know that this register is
10628 // set.
10629 OpInfo.AssignedRegs.AddInlineAsmOperands(
10630 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10632 false, 0, DL, DAG, Info.AsmNodeOperands);
10633 }
10634 break;
10635
10636 case InlineAsm::isInput:
10637 case InlineAsm::isLabel: {
10638 SDValue InOperandVal = OpInfo.CallOperand;
10639
10640 if (OpInfo.isMatchingInputConstraint()) {
10641 // If this is required to match an output register we have already set,
10642 // just use its register.
10643 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
10644 Info.AsmNodeOperands);
10645 InlineAsm::Flag Flag(Info.AsmNodeOperands[CurOp]->getAsZExtVal());
10646 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10647 if (OpInfo.isIndirect) {
10648 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10649 Info.ErrorMsg << "inline asm not supported yet: cannot handle "
10650 << "tied indirect register inputs";
10651 return true;
10652 }
10653
10658 auto *R = cast<RegisterSDNode>(Info.AsmNodeOperands[CurOp + 1]);
10659 Register TiedReg = R->getReg();
10660 MVT RegVT = R->getSimpleValueType(0);
10661 const TargetRegisterClass *RC;
10662 if (TiedReg.isVirtual()) {
10663 RC = MRI.getRegClass(TiedReg);
10664 } else if (RegVT == MVT::Untyped) {
10665 RC = TRI.getMinimalPhysRegClass(TiedReg);
10666 } else {
10667 RC = TLI.getRegClassFor(RegVT);
10668 if (!RC->contains(TiedReg)) {
10669 const TargetRegisterClass *PhysRC =
10670 TRI.getMinimalPhysRegClass(TiedReg);
10671 if (PhysRC->isAllocatable())
10672 RC = PhysRC;
10673 }
10674 }
10675
10676 for (unsigned I = 0, E = Flag.getNumOperandRegisters(); I != E; ++I)
10677 Regs.push_back(MRI.createVirtualRegister(RC));
10678
10679 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10680
10681 // Use the produced MatchedRegs object to
10682 MatchedRegs.getCopyToRegs(InOperandVal, DAG, DL, Info.Chain,
10683 &Info.Glue, &Call);
10685 OpInfo.getMatchedOperand(), DL, DAG,
10686 Info.AsmNodeOperands);
10687 break;
10688 }
10689
10690 assert(Flag.isMemKind() && "Unknown matching constraint!");
10691 assert(Flag.getNumOperandRegisters() == 1 &&
10692 "Unexpected number of operands");
10693
10694 // Add information to the INLINEASM node to know about this input.
10695 // See InlineAsm.h isUseOperandTiedToDef.
10696 Flag.clearMemConstraint();
10697 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10698 Info.AsmNodeOperands.push_back(DAG.getTargetConstant(
10699 Flag, DL, TLI.getPointerTy(DAG.getDataLayout())));
10700 Info.AsmNodeOperands.push_back(Info.AsmNodeOperands[CurOp + 1]);
10701 break;
10702 }
10703
10704 // Treat indirect 'X' constraint as memory.
10705 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10706 OpInfo.isIndirect)
10707 OpInfo.ConstraintType = TargetLowering::C_Memory;
10708
10709 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10710 OpInfo.ConstraintType == TargetLowering::C_Other) {
10711 std::vector<SDValue> Ops;
10712 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
10713 Ops, DAG);
10714 if (Ops.empty()) {
10715 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10716 if (isa<ConstantSDNode>(InOperandVal)) {
10717 Info.ErrorMsg << "value out of range for constraint '"
10718 << OpInfo.ConstraintCode << "'";
10719 return true;
10720 }
10721
10722 Info.ErrorMsg << "invalid operand for inline asm constraint '"
10723 << OpInfo.ConstraintCode << "'";
10724 return true;
10725 }
10726
10727 // Add information to the INLINEASM node to know about this input.
10728 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10729 Info.AsmNodeOperands.push_back(DAG.getTargetConstant(
10730 ResOpType, DL, TLI.getPointerTy(DAG.getDataLayout())));
10731 llvm::append_range(Info.AsmNodeOperands, Ops);
10732 break;
10733 }
10734
10735 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10736 assert((OpInfo.isIndirect ||
10737 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10738 "Operand must be indirect to be a mem!");
10739
10740 assert(InOperandVal.getValueType() ==
10741 TLI.getPointerTy(
10742 DAG.getDataLayout(),
10743 OpInfo.CallOperandVal
10744 ? cast<PointerType>(OpInfo.CallOperandVal->getType())
10745 ->getAddressSpace()
10746 : 0) &&
10747 "Memory operands expect pointer values");
10748
10749 const InlineAsm::ConstraintCode ConstraintID =
10750 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10752 "Failed to convert memory constraint code to constraint id.");
10753
10754 // Add information to the INLINEASM node to know about this input.
10756 ResOpType.setMemConstraint(ConstraintID);
10757 Info.AsmNodeOperands.push_back(
10758 DAG.getTargetConstant(ResOpType, DL, MVT::i32));
10759 Info.AsmNodeOperands.push_back(InOperandVal);
10760 break;
10761 }
10762
10763 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10764 const InlineAsm::ConstraintCode ConstraintID =
10765 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10767 "Failed to convert memory constraint code to constraint id.");
10768
10770
10771 SDValue AsmOp = InOperandVal;
10772 if (TLI.supportsInlineAsmFuncOperands() && isFunction(InOperandVal)) {
10773 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
10774 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10775 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
10776 InOperandVal.getValueType(),
10777 GA->getOffset());
10778 }
10779
10780 // Add information to the INLINEASM node to know about this input.
10781 ResOpType.setMemConstraint(ConstraintID);
10782
10783 Info.AsmNodeOperands.push_back(
10784 DAG.getTargetConstant(ResOpType, DL, MVT::i32));
10785 Info.AsmNodeOperands.push_back(AsmOp);
10786 break;
10787 }
10788
10789 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10790 OpInfo.ConstraintType != TargetLowering::C_Register) {
10791 Info.ErrorMsg << "unknown asm constraint '" << OpInfo.ConstraintCode
10792 << "'";
10793 return true;
10794 }
10795
10796 // TODO: Support this.
10797 if (OpInfo.isIndirect) {
10798 Info.ErrorMsg << "cannot handle indirect register inputs yet for "
10799 << "constraint '" << OpInfo.ConstraintCode << "'";
10800 return true;
10801 }
10802
10803 // Copy the input into the appropriate registers.
10804 if (OpInfo.AssignedRegs.Regs.empty()) {
10805 Info.ErrorMsg << "could not allocate input reg for constraint '"
10806 << OpInfo.ConstraintCode << "'";
10807 return true;
10808 }
10809
10810 if (DetectWriteToReservedRegister())
10811 return true;
10812
10813 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, DL, Info.Chain,
10814 &Info.Glue, &Call);
10815 OpInfo.AssignedRegs.AddInlineAsmOperands(
10816 InlineAsm::Kind::RegUse, false, 0, DL, DAG, Info.AsmNodeOperands);
10817 break;
10818 }
10819
10821 // Add the clobbered value to the operand list, so that the register
10822 // allocator is aware that the physreg got clobbered.
10823 if (!OpInfo.AssignedRegs.Regs.empty())
10824 OpInfo.AssignedRegs.AddInlineAsmOperands(
10825 InlineAsm::Kind::Clobber, false, 0, DL, DAG, Info.AsmNodeOperands);
10826 break;
10827 }
10828 }
10829
10830 return false;
10831}
10832
10833/// DetermineConstraints - Find the constraints to use for inline asm operands.
10834static bool
10835determineConstraints(ConstraintDecisionInfo &Info,
10836 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10837 const CallBase &Call, SelectionDAGBuilder &Builder,
10838 const TargetLowering &TLI, const TargetMachine &TM,
10839 SelectionDAG &DAG, const BasicBlock *EHPadBB) {
10840 const auto *IA = cast<InlineAsm>(Call.getCalledOperand());
10841 ExtraFlags ExtraInfo(Call);
10842
10843 // First pass: Construct operand info objects.
10844 Info.HasSideEffect = IA->hasSideEffects();
10845 if (constructOperandInfo(Info, TargetConstraints, Builder, TLI, ExtraInfo))
10846 return true;
10847
10848 // We won't need to flush pending loads if this asm doesn't touch
10849 // memory and is nonvolatile.
10850 Info.Chain = Info.HasSideEffect ? Builder.getRoot() : DAG.getRoot();
10851
10852 bool IsCallBr = isa<CallBrInst>(Call);
10853 bool EmitEHLabels = isa<InvokeInst>(Call);
10854 if (IsCallBr || EmitEHLabels)
10855 // If this is a callbr or invoke we need to flush pending exports since
10856 // inlineasm_br and invoke are terminators.
10857 // We need to do this before nodes are glued to the inlineasm_br node.
10858 Info.Chain = Builder.getControlRoot();
10859
10860 if (EmitEHLabels)
10861 Info.Chain = Builder.lowerStartEH(Info.Chain, EHPadBB, Info.BeginLabel);
10862
10863 // Second pass: Compute which constraint option to use.
10864 computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM, DAG);
10865
10866 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10867 Info.AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
10868 Info.AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
10869 IA->getAsmString().data(), TLI.getProgramPointerTy(DAG.getDataLayout())));
10870
10871 // If we have a !srcloc metadata node associated with it, we want to attach
10872 // this to the ultimately generated inline asm machineinstr. To do this, we
10873 // pass in the third operand as this (potentially null) inline asm MDNode.
10874 const MDNode *SrcLoc = Call.getMetadata("srcloc");
10875 Info.AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
10876
10877 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10878 // bits as operand 3.
10879 Info.AsmNodeOperands.push_back(
10880 DAG.getTargetConstant(ExtraInfo.get(), Builder.getCurSDLoc(),
10881 TLI.getPointerTy(DAG.getDataLayout())));
10882
10883 // Third pass: Prepare DAG-level operands
10884 return prepareDAGLevelOperands(Info, Call, Builder, TLI, DAG);
10885}
10886
10887/// visitInlineAsm - Handle a call to an InlineAsm object.
10888void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
10889 const BasicBlock *EHPadBB) {
10890 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10892 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);
10893
10894 assert((!isa<InvokeInst>(Call) || EHPadBB) &&
10895 "InvokeInst must have an EHPadBB");
10896
10897 ConstraintDecisionInfo Info;
10898 if (determineConstraints(Info, TargetConstraints, Call, *this, TLI, TM, DAG,
10899 EHPadBB))
10900 return emitInlineAsmError(Call, Info.ErrorMsg.str());
10901
10902 SDValue Glue = Info.Glue;
10903 SDValue Chain = Info.Chain;
10904
10905 // Finish up input operands. Set the input chain and add the flag last.
10906 Info.AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10907 if (Glue.getNode())
10908 Info.AsmNodeOperands.push_back(Glue);
10909
10910 bool IsCallBr = isa<CallBrInst>(Call);
10911 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10912 Chain =
10913 DAG.getNode(ISDOpc, getCurSDLoc(), DAG.getVTList(MVT::Other, MVT::Glue),
10914 Info.AsmNodeOperands);
10915 Glue = Chain.getValue(1);
10916
10917 // Do additional work to generate outputs.
10918
10919 SmallVector<EVT, 1> ResultVTs;
10920 SmallVector<SDValue, 1> ResultValues;
10921 SmallVector<SDValue, 8> OutChains;
10922
10923 llvm::Type *CallResultType = Call.getType();
10924 ArrayRef<Type *> ResultTypes;
10925 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
10926 ResultTypes = StructResult->elements();
10927 else if (!CallResultType->isVoidTy())
10928 ResultTypes = ArrayRef(CallResultType);
10929
10930 auto CurResultType = ResultTypes.begin();
10931 auto handleRegAssign = [&](SDValue V) {
10932 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10933 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10934 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
10935 ++CurResultType;
10936 // If the type of the inline asm call site return value is different but has
10937 // same size as the type of the asm output bitcast it. One example of this
10938 // is for vectors with different width / number of elements. This can
10939 // happen for register classes that can contain multiple different value
10940 // types. The preg or vreg allocated may not have the same VT as was
10941 // expected.
10942 //
10943 // This can also happen for a return value that disagrees with the register
10944 // class it is put in, eg. a double in a general-purpose register on a
10945 // 32-bit machine.
10946 if (ResultVT != V.getValueType() &&
10947 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10948 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
10949 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10950 V.getValueType().isInteger()) {
10951 // If a result value was tied to an input value, the computed result
10952 // may have a wider width than the expected result. Extract the
10953 // relevant portion.
10954 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
10955 }
10956 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10957 ResultVTs.push_back(ResultVT);
10958 ResultValues.push_back(V);
10959 };
10960
10961 // Deal with output operands.
10962 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10963 if (OpInfo.Type == InlineAsm::isOutput) {
10964 SDValue Val;
10965 // Skip trivial output operands.
10966 if (OpInfo.AssignedRegs.Regs.empty())
10967 continue;
10968
10969 switch (OpInfo.ConstraintType) {
10972 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
10973 Chain, &Glue, &Call);
10974 break;
10977 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
10978 OpInfo, DAG);
10979 break;
10981 break; // Already handled.
10983 break; // Silence warning.
10985 assert(false && "Unexpected unknown constraint");
10986 }
10987
10988 // Indirect output manifest as stores. Record output chains.
10989 if (OpInfo.isIndirect) {
10990 const Value *Ptr = OpInfo.CallOperandVal;
10991 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10992 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
10993 MachinePointerInfo(Ptr));
10994 OutChains.push_back(Store);
10995 } else {
10996 // generate CopyFromRegs to associated registers.
10997 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10998 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10999 for (const SDValue &V : Val->op_values())
11000 handleRegAssign(V);
11001 } else
11002 handleRegAssign(Val);
11003 }
11004 }
11005 }
11006
11007 // Set results.
11008 if (!ResultValues.empty()) {
11009 assert(CurResultType == ResultTypes.end() &&
11010 "Mismatch in number of ResultTypes");
11011 assert(ResultValues.size() == ResultTypes.size() &&
11012 "Mismatch in number of output operands in asm result");
11013
11015 DAG.getVTList(ResultVTs), ResultValues);
11016 setValue(&Call, V);
11017 }
11018
11019 // Collect store chains.
11020 if (!OutChains.empty())
11021 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
11022
11023 if (const auto *II = dyn_cast<InvokeInst>(&Call))
11024 Chain = lowerEndEH(Chain, II, EHPadBB, Info.BeginLabel);
11025
11026 // Only Update Root if inline assembly has a memory effect.
11027 if (ResultValues.empty() || Info.HasSideEffect || !OutChains.empty() ||
11028 IsCallBr || isa<InvokeInst>(Call))
11029 DAG.setRoot(Chain);
11030}
11031
11032void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
11033 const Twine &Message) {
11034 LLVMContext &Ctx = *DAG.getContext();
11035 Ctx.diagnose(DiagnosticInfoInlineAsm(Call, Message));
11036
11037 // Make sure we leave the DAG in a valid state
11038 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11039 SmallVector<EVT, 1> ValueVTs;
11040 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
11041
11042 if (ValueVTs.empty())
11043 return;
11044
11046 for (const EVT &VT : ValueVTs)
11047 Ops.push_back(DAG.getUNDEF(VT));
11048
11049 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
11050}
11051
11052void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
11053 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
11054 MVT::Other, getRoot(),
11055 getValue(I.getArgOperand(0)),
11056 DAG.getSrcValue(I.getArgOperand(0))));
11057}
11058
11059void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
11060 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11061 const DataLayout &DL = DAG.getDataLayout();
11062 SDValue V = DAG.getVAArg(
11063 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
11064 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
11065 DL.getABITypeAlign(I.getType()).value());
11066 DAG.setRoot(V.getValue(1));
11067
11068 if (I.getType()->isPointerTy())
11069 V = DAG.getPtrExtOrTrunc(
11070 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
11071 setValue(&I, V);
11072}
11073
11074void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
11075 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
11076 MVT::Other, getRoot(),
11077 getValue(I.getArgOperand(0)),
11078 DAG.getSrcValue(I.getArgOperand(0))));
11079}
11080
11081void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
11082 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
11083 MVT::Other, getRoot(),
11084 getValue(I.getArgOperand(0)),
11085 getValue(I.getArgOperand(1)),
11086 DAG.getSrcValue(I.getArgOperand(0)),
11087 DAG.getSrcValue(I.getArgOperand(1))));
11088}
11089
11091 const Instruction &I,
11092 SDValue Op) {
11093 std::optional<ConstantRange> CR = getRange(I);
11094
11095 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
11096 return Op;
11097
11098 APInt Hi = CR->getUnsignedMax();
11099 unsigned Bits = std::max(Hi.getActiveBits(),
11100 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
11101
11102 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
11103
11104 SDLoc SL = getCurSDLoc();
11105
11106 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
11107 DAG.getValueType(SmallVT));
11108 unsigned NumVals = Op.getNode()->getNumValues();
11109 if (NumVals == 1)
11110 return ZExt;
11111
11113
11114 Ops.push_back(ZExt);
11115 for (unsigned I = 1; I != NumVals; ++I)
11116 Ops.push_back(Op.getValue(I));
11117
11118 return DAG.getMergeValues(Ops, SL);
11119}
11120
11122 SelectionDAG &DAG, const Instruction &I, SDValue Op) {
11123 FPClassTest Classes = getNoFPClass(I);
11124 if (Classes == fcNone)
11125 return Op;
11126
11127 SDLoc SL = getCurSDLoc();
11128 SDValue TestConst = DAG.getTargetConstant(Classes, SDLoc(), MVT::i32);
11129
11130 if (Op.getOpcode() != ISD::MERGE_VALUES) {
11131 return DAG.getNode(ISD::AssertNoFPClass, SL, Op.getValueType(), Op,
11132 TestConst);
11133 }
11134
11135 SmallVector<SDValue, 8> Ops(Op.getNumOperands());
11136 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
11137 SDValue MergeOp = Op.getOperand(I);
11138 Ops[I] = DAG.getNode(ISD::AssertNoFPClass, SL, MergeOp.getValueType(),
11139 MergeOp, TestConst);
11140 }
11141
11142 return DAG.getMergeValues(Ops, SL);
11143}
11144
11145/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
11146/// the call being lowered.
11147///
11148/// This is a helper for lowering intrinsics that follow a target calling
11149/// convention or require stack pointer adjustment. Only a subset of the
11150/// intrinsic's operands need to participate in the calling convention.
11153 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
11154 AttributeSet RetAttrs, bool IsPatchPoint) {
11156 Args.reserve(NumArgs);
11157
11158 // Populate the argument list.
11159 // Attributes for args start at offset 1, after the return attribute.
11160 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
11161 ArgI != ArgE; ++ArgI) {
11162 const Value *V = Call->getOperand(ArgI);
11163
11164 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
11165
11166 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
11167 Entry.setAttributes(Call, ArgI);
11168 Args.push_back(Entry);
11169 }
11170
11172 .setChain(getRoot())
11173 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
11174 RetAttrs)
11175 .setDiscardResult(Call->use_empty())
11176 .setIsPatchPoint(IsPatchPoint)
11178 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
11179}
11180
11181/// Add a stack map intrinsic call's live variable operands to a stackmap
11182/// or patchpoint target node's operand list.
11183///
11184/// Constants are converted to TargetConstants purely as an optimization to
11185/// avoid constant materialization and register allocation.
11186///
11187/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
11188/// generate addess computation nodes, and so FinalizeISel can convert the
11189/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
11190/// address materialization and register allocation, but may also be required
11191/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
11192/// alloca in the entry block, then the runtime may assume that the alloca's
11193/// StackMap location can be read immediately after compilation and that the
11194/// location is valid at any point during execution (this is similar to the
11195/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
11196/// only available in a register, then the runtime would need to trap when
11197/// execution reaches the StackMap in order to read the alloca's location.
11198static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
11200 SelectionDAGBuilder &Builder) {
11201 SelectionDAG &DAG = Builder.DAG;
11202 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
11203 SDValue Op = Builder.getValue(Call.getArgOperand(I));
11204
11205 // Things on the stack are pointer-typed, meaning that they are already
11206 // legal and can be emitted directly to target nodes.
11208 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
11209 } else {
11210 // Otherwise emit a target independent node to be legalised.
11211 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
11212 }
11213 }
11214}
11215
11216/// Lower llvm.experimental.stackmap.
11217void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
11218 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
11219 // [live variables...])
11220
11221 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
11222
11223 SDValue Chain, InGlue, Callee;
11225
11226 SDLoc DL = getCurSDLoc();
11228
11229 // The stackmap intrinsic only records the live variables (the arguments
11230 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
11231 // intrinsic, this won't be lowered to a function call. This means we don't
11232 // have to worry about calling conventions and target specific lowering code.
11233 // Instead we perform the call lowering right here.
11234 //
11235 // chain, flag = CALLSEQ_START(chain, 0, 0)
11236 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
11237 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
11238 //
11239 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
11240 InGlue = Chain.getValue(1);
11241
11242 // Add the STACKMAP operands, starting with DAG house-keeping.
11243 Ops.push_back(Chain);
11244 Ops.push_back(InGlue);
11245
11246 // Add the <id>, <numShadowBytes> operands.
11247 //
11248 // These do not require legalisation, and can be emitted directly to target
11249 // constant nodes.
11251 assert(ID.getValueType() == MVT::i64);
11252 SDValue IDConst =
11253 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
11254 Ops.push_back(IDConst);
11255
11256 SDValue Shad = getValue(CI.getArgOperand(1));
11257 assert(Shad.getValueType() == MVT::i32);
11258 SDValue ShadConst =
11259 DAG.getTargetConstant(Shad->getAsZExtVal(), DL, Shad.getValueType());
11260 Ops.push_back(ShadConst);
11261
11262 // Add the live variables.
11263 addStackMapLiveVars(CI, 2, DL, Ops, *this);
11264
11265 // Create the STACKMAP node.
11266 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
11267 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
11268 InGlue = Chain.getValue(1);
11269
11270 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
11271
11272 // Stackmaps don't generate values, so nothing goes into the NodeMap.
11273
11274 // Set the root to the target-lowered call chain.
11275 DAG.setRoot(Chain);
11276
11277 // Inform the Frame Information that we have a stackmap in this function.
11278 FuncInfo.MF->getFrameInfo().setHasStackMap();
11279}
11280
11281/// Lower llvm.experimental.patchpoint directly to its target opcode.
11282void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
11283 const BasicBlock *EHPadBB) {
11284 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
11285 // i32 <numBytes>,
11286 // i8* <target>,
11287 // i32 <numArgs>,
11288 // [Args...],
11289 // [live variables...])
11290
11292 bool IsAnyRegCC = CC == CallingConv::AnyReg;
11293 bool HasDef = !CB.getType()->isVoidTy();
11294 SDLoc dl = getCurSDLoc();
11296
11297 // Handle immediate and symbolic callees.
11298 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
11299 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
11300 /*isTarget=*/true);
11301 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
11302 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
11303 SDLoc(SymbolicCallee),
11304 SymbolicCallee->getValueType(0));
11305
11306 // Get the real number of arguments participating in the call <numArgs>
11308 unsigned NumArgs = NArgVal->getAsZExtVal();
11309
11310 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
11311 // Intrinsics include all meta-operands up to but not including CC.
11312 unsigned NumMetaOpers = PatchPointOpers::CCPos;
11313 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
11314 "Not enough arguments provided to the patchpoint intrinsic");
11315
11316 // For AnyRegCC the arguments are lowered later on manually.
11317 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11318 Type *ReturnTy =
11319 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
11320
11321 TargetLowering::CallLoweringInfo CLI(DAG);
11322 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
11323 ReturnTy, CB.getAttributes().getRetAttrs(), true);
11324 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
11325
11326 SDNode *CallEnd = Result.second.getNode();
11327 if (CallEnd->getOpcode() == ISD::EH_LABEL)
11328 CallEnd = CallEnd->getOperand(0).getNode();
11329 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
11330 CallEnd = CallEnd->getOperand(0).getNode();
11331
11332 /// Get a call instruction from the call sequence chain.
11333 /// Tail calls are not allowed.
11334 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
11335 "Expected a callseq node.");
11336 SDNode *Call = CallEnd->getOperand(0).getNode();
11337 bool HasGlue = Call->getGluedNode();
11338
11339 // Replace the target specific call node with the patchable intrinsic.
11341
11342 // Push the chain.
11343 Ops.push_back(*(Call->op_begin()));
11344
11345 // Optionally, push the glue (if any).
11346 if (HasGlue)
11347 Ops.push_back(*(Call->op_end() - 1));
11348
11349 // Push the register mask info.
11350 if (HasGlue)
11351 Ops.push_back(*(Call->op_end() - 2));
11352 else
11353 Ops.push_back(*(Call->op_end() - 1));
11354
11355 // Add the <id> and <numBytes> constants.
11357 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
11359 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
11360
11361 // Add the callee.
11362 Ops.push_back(Callee);
11363
11364 // Adjust <numArgs> to account for any arguments that have been passed on the
11365 // stack instead.
11366 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
11367 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
11368 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11369 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
11370
11371 // Add the calling convention
11372 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
11373
11374 // Add the arguments we omitted previously. The register allocator should
11375 // place these in any free register.
11376 if (IsAnyRegCC)
11377 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
11378 Ops.push_back(getValue(CB.getArgOperand(i)));
11379
11380 // Push the arguments from the call instruction.
11381 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
11382 Ops.append(Call->op_begin() + 2, e);
11383
11384 // Push live variables for the stack map.
11385 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
11386
11387 SDVTList NodeTys;
11388 if (IsAnyRegCC && HasDef) {
11389 // Create the return types based on the intrinsic definition
11390 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11391 SmallVector<EVT, 3> ValueVTs;
11392 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
11393 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
11394
11395 // There is always a chain and a glue type at the end
11396 ValueVTs.push_back(MVT::Other);
11397 ValueVTs.push_back(MVT::Glue);
11398 NodeTys = DAG.getVTList(ValueVTs);
11399 } else
11400 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
11401
11402 // Replace the target specific call node with a PATCHPOINT node.
11403 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
11404
11405 // Update the NodeMap.
11406 if (HasDef) {
11407 if (IsAnyRegCC)
11408 setValue(&CB, SDValue(PPV.getNode(), 0));
11409 else
11410 setValue(&CB, Result.first);
11411 }
11412
11413 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
11414 // call sequence. Furthermore the location of the chain and glue can change
11415 // when the AnyReg calling convention is used and the intrinsic returns a
11416 // value.
11417 if (IsAnyRegCC && HasDef) {
11418 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
11419 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
11420 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
11421 } else
11422 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
11423 DAG.DeleteNode(Call);
11424
11425 // Inform the Frame Information that we have a patchpoint in this function.
11426 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11427}
11428
11429void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
11430 unsigned Intrinsic) {
11431 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11432 SDValue Op1 = getValue(I.getArgOperand(0));
11433 SDValue Op2;
11434 if (I.arg_size() > 1)
11435 Op2 = getValue(I.getArgOperand(1));
11436 SDLoc dl = getCurSDLoc();
11437 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11438 SDValue Res;
11439 SDNodeFlags SDFlags;
11440 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
11441 SDFlags.copyFMF(*FPMO);
11442
11443 switch (Intrinsic) {
11444 case Intrinsic::vector_reduce_fadd:
11445 if (SDFlags.hasAllowReassociation())
11446 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
11447 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
11448 SDFlags);
11449 else
11450 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
11451 break;
11452 case Intrinsic::vector_reduce_fmul:
11453 if (SDFlags.hasAllowReassociation())
11454 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
11455 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
11456 SDFlags);
11457 else
11458 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
11459 break;
11460 case Intrinsic::vector_reduce_add:
11461 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
11462 break;
11463 case Intrinsic::vector_reduce_mul:
11464 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
11465 break;
11466 case Intrinsic::vector_reduce_and:
11467 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
11468 break;
11469 case Intrinsic::vector_reduce_or:
11470 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
11471 break;
11472 case Intrinsic::vector_reduce_xor:
11473 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
11474 break;
11475 case Intrinsic::vector_reduce_smax:
11476 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
11477 break;
11478 case Intrinsic::vector_reduce_smin:
11479 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
11480 break;
11481 case Intrinsic::vector_reduce_umax:
11482 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
11483 break;
11484 case Intrinsic::vector_reduce_umin:
11485 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
11486 break;
11487 case Intrinsic::vector_reduce_fmax:
11488 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
11489 break;
11490 case Intrinsic::vector_reduce_fmin:
11491 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
11492 break;
11493 case Intrinsic::vector_reduce_fmaximum:
11494 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
11495 break;
11496 case Intrinsic::vector_reduce_fminimum:
11497 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
11498 break;
11499 case Intrinsic::vector_reduce_fmaximumnum:
11500 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUMNUM, dl, VT, Op1, SDFlags);
11501 break;
11502 case Intrinsic::vector_reduce_fminimumnum:
11503 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUMNUM, dl, VT, Op1, SDFlags);
11504 break;
11505 default:
11506 llvm_unreachable("Unhandled vector reduce intrinsic");
11507 }
11508 setValue(&I, Res);
11509}
11510
11511/// Returns an AttributeList representing the attributes applied to the return
11512/// value of the given call.
11515 if (CLI.RetSExt)
11516 Attrs.push_back(Attribute::SExt);
11517 if (CLI.RetZExt)
11518 Attrs.push_back(Attribute::ZExt);
11519 if (CLI.IsInReg)
11520 Attrs.push_back(Attribute::InReg);
11521
11522 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
11523 Attrs);
11524}
11525
11526/// TargetLowering::LowerCallTo - This is the default LowerCallTo
11527/// implementation, which just calls LowerCall.
11528/// FIXME: When all targets are
11529/// migrated to using LowerCall, this hook should be integrated into SDISel.
11530std::pair<SDValue, SDValue>
11532 LLVMContext &Context = CLI.RetTy->getContext();
11533
11534 // Handle the incoming return values from the call.
11535 CLI.Ins.clear();
11536 SmallVector<Type *, 4> RetOrigTys;
11538 auto &DL = CLI.DAG.getDataLayout();
11539 ComputeValueTypes(DL, CLI.OrigRetTy, RetOrigTys, &Offsets);
11540
11541 SmallVector<EVT, 4> RetVTs;
11542 if (CLI.RetTy != CLI.OrigRetTy) {
11543 assert(RetOrigTys.size() == 1 &&
11544 "Only supported for non-aggregate returns");
11545 RetVTs.push_back(getValueType(DL, CLI.RetTy));
11546 } else {
11547 for (Type *Ty : RetOrigTys)
11548 RetVTs.push_back(getValueType(DL, Ty));
11549 }
11550
11551 if (CLI.IsPostTypeLegalization) {
11552 // If we are lowering a libcall after legalization, split the return type.
11553 SmallVector<Type *, 4> OldRetOrigTys;
11554 SmallVector<EVT, 4> OldRetVTs;
11555 SmallVector<TypeSize, 4> OldOffsets;
11556 RetOrigTys.swap(OldRetOrigTys);
11557 RetVTs.swap(OldRetVTs);
11558 Offsets.swap(OldOffsets);
11559
11560 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
11561 EVT RetVT = OldRetVTs[i];
11562 uint64_t Offset = OldOffsets[i];
11563 MVT RegisterVT = getRegisterType(Context, RetVT);
11564 unsigned NumRegs = getNumRegisters(Context, RetVT);
11565 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
11566 RetOrigTys.append(NumRegs, OldRetOrigTys[i]);
11567 RetVTs.append(NumRegs, RegisterVT);
11568 for (unsigned j = 0; j != NumRegs; ++j)
11569 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
11570 }
11571 }
11572
11574 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
11575
11576 bool CanLowerReturn =
11578 CLI.IsVarArg, Outs, Context, CLI.RetTy);
11579
11580 SDValue DemoteStackSlot;
11581 int DemoteStackIdx = -100;
11582 if (!CanLowerReturn) {
11583 // FIXME: equivalent assert?
11584 // assert(!CS.hasInAllocaArgument() &&
11585 // "sret demotion is incompatible with inalloca");
11586 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
11587 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
11589 DemoteStackIdx =
11590 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
11591 Type *StackSlotPtrType = PointerType::get(Context, DL.getAllocaAddrSpace());
11592
11593 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
11594 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11595 Entry.IsSRet = true;
11596 Entry.Alignment = Alignment;
11597 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
11598 CLI.NumFixedArgs += 1;
11599 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11600 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(Context);
11601
11602 // sret demotion isn't compatible with tail-calls, since the sret argument
11603 // points into the callers stack frame.
11604 CLI.IsTailCall = false;
11605 } else {
11606 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11607 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
11608 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
11609 ISD::ArgFlagsTy Flags;
11610 if (NeedsRegBlock) {
11611 Flags.setInConsecutiveRegs();
11612 if (I == RetVTs.size() - 1)
11613 Flags.setInConsecutiveRegsLast();
11614 }
11615 EVT VT = RetVTs[I];
11616 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11617 unsigned NumRegs =
11618 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11619 for (unsigned i = 0; i != NumRegs; ++i) {
11620 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
11622 if (CLI.RetTy->isPointerTy()) {
11623 Ret.Flags.setPointer();
11625 cast<PointerType>(CLI.RetTy)->getAddressSpace());
11626 }
11627 if (CLI.RetSExt)
11628 Ret.Flags.setSExt();
11629 if (CLI.RetZExt)
11630 Ret.Flags.setZExt();
11631 if (CLI.IsInReg)
11632 Ret.Flags.setInReg();
11633 CLI.Ins.push_back(Ret);
11634 }
11635 }
11636 }
11637
11638 // We push in swifterror return as the last element of CLI.Ins.
11639 ArgListTy &Args = CLI.getArgs();
11640 if (supportSwiftError()) {
11641 for (const ArgListEntry &Arg : Args) {
11642 if (Arg.IsSwiftError) {
11643 ISD::ArgFlagsTy Flags;
11644 Flags.setSwiftError();
11646 PointerType::getUnqual(Context),
11647 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
11648 CLI.Ins.push_back(Ret);
11649 }
11650 }
11651 }
11652
11653 // Handle all of the outgoing arguments.
11654 CLI.Outs.clear();
11655 CLI.OutVals.clear();
11656 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11657 SmallVector<Type *, 4> OrigArgTys;
11658 ComputeValueTypes(DL, Args[i].OrigTy, OrigArgTys);
11659 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11660 Type *FinalType = Args[i].Ty;
11661 if (Args[i].IsByVal)
11662 FinalType = Args[i].IndirectType;
11663 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11664 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
11665 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
11666 ++Value) {
11667 Type *OrigArgTy = OrigArgTys[Value];
11668 Type *ArgTy = OrigArgTy;
11669 if (Args[i].Ty != Args[i].OrigTy) {
11670 assert(Value == 0 && "Only supported for non-aggregate arguments");
11671 ArgTy = Args[i].Ty;
11672 }
11673
11674 EVT VT = getValueType(DL, ArgTy);
11675 SDValue Op = SDValue(Args[i].Node.getNode(),
11676 Args[i].Node.getResNo() + Value);
11677 ISD::ArgFlagsTy Flags;
11678
11679 // Certain targets (such as MIPS), may have a different ABI alignment
11680 // for a type depending on the context. Give the target a chance to
11681 // specify the alignment it wants.
11682 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11683 Flags.setOrigAlign(OriginalAlignment);
11684
11685 if (i >= CLI.NumFixedArgs)
11686 Flags.setVarArg();
11687 if (ArgTy->isPointerTy()) {
11688 Flags.setPointer();
11689 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
11690 }
11691 if (Args[i].IsZExt)
11692 Flags.setZExt();
11693 if (Args[i].IsSExt)
11694 Flags.setSExt();
11695 if (Args[i].IsNoExt)
11696 Flags.setNoExt();
11697 if (Args[i].IsInReg) {
11698 // If we are using vectorcall calling convention, a structure that is
11699 // passed InReg - is surely an HVA
11701 isa<StructType>(FinalType)) {
11702 // The first value of a structure is marked
11703 if (0 == Value)
11704 Flags.setHvaStart();
11705 Flags.setHva();
11706 }
11707 // Set InReg Flag
11708 Flags.setInReg();
11709 }
11710 if (Args[i].IsSRet)
11711 Flags.setSRet();
11712 if (Args[i].IsSwiftSelf)
11713 Flags.setSwiftSelf();
11714 if (Args[i].IsSwiftAsync)
11715 Flags.setSwiftAsync();
11716 if (Args[i].IsSwiftError)
11717 Flags.setSwiftError();
11718 if (Args[i].IsCFGuardTarget)
11719 Flags.setCFGuardTarget();
11720 if (Args[i].IsByVal)
11721 Flags.setByVal();
11722 if (Args[i].IsByRef)
11723 Flags.setByRef();
11724 if (Args[i].IsPreallocated) {
11725 Flags.setPreallocated();
11726 // Set the byval flag for CCAssignFn callbacks that don't know about
11727 // preallocated. This way we can know how many bytes we should've
11728 // allocated and how many bytes a callee cleanup function will pop. If
11729 // we port preallocated to more targets, we'll have to add custom
11730 // preallocated handling in the various CC lowering callbacks.
11731 Flags.setByVal();
11732 }
11733 if (Args[i].IsInAlloca) {
11734 Flags.setInAlloca();
11735 // Set the byval flag for CCAssignFn callbacks that don't know about
11736 // inalloca. This way we can know how many bytes we should've allocated
11737 // and how many bytes a callee cleanup function will pop. If we port
11738 // inalloca to more targets, we'll have to add custom inalloca handling
11739 // in the various CC lowering callbacks.
11740 Flags.setByVal();
11741 }
11742 Align MemAlign;
11743 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11744 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
11745 Flags.setByValSize(FrameSize);
11746
11747 // info is not there but there are cases it cannot get right.
11748 if (auto MA = Args[i].Alignment)
11749 MemAlign = *MA;
11750 else
11751 MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL);
11752 } else if (auto MA = Args[i].Alignment) {
11753 MemAlign = *MA;
11754 } else {
11755 MemAlign = OriginalAlignment;
11756 }
11757 Flags.setMemAlign(MemAlign);
11758 if (Args[i].IsNest)
11759 Flags.setNest();
11760 if (NeedsRegBlock)
11761 Flags.setInConsecutiveRegs();
11762
11763 MVT PartVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11764 unsigned NumParts =
11765 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11768
11769 if (Args[i].IsSExt)
11771 else if (Args[i].IsZExt)
11773
11774 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11775 // for now.
11776 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11778 assert((CLI.RetTy == Args[i].Ty ||
11779 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11781 Args[i].Ty->getPointerAddressSpace())) &&
11782 RetVTs.size() == NumValues && "unexpected use of 'returned'");
11783 // Before passing 'returned' to the target lowering code, ensure that
11784 // either the register MVT and the actual EVT are the same size or that
11785 // the return value and argument are extended in the same way; in these
11786 // cases it's safe to pass the argument register value unchanged as the
11787 // return register value (although it's at the target's option whether
11788 // to do so)
11789 // TODO: allow code generation to take advantage of partially preserved
11790 // registers rather than clobbering the entire register when the
11791 // parameter extension method is not compatible with the return
11792 // extension method
11793 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11794 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11795 CLI.RetZExt == Args[i].IsZExt))
11796 Flags.setReturned();
11797 }
11798
11799 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
11800 CLI.CallConv, ExtendKind);
11801
11802 for (unsigned j = 0; j != NumParts; ++j) {
11803 // if it isn't first piece, alignment must be 1
11804 // For scalable vectors the scalable part is currently handled
11805 // by individual targets, so we just use the known minimum size here.
11806 ISD::OutputArg MyFlags(
11807 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
11808 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11809 if (NumParts > 1 && j == 0)
11810 MyFlags.Flags.setSplit();
11811 else if (j != 0) {
11812 MyFlags.Flags.setOrigAlign(Align(1));
11813 if (j == NumParts - 1)
11814 MyFlags.Flags.setSplitEnd();
11815 }
11816
11817 CLI.Outs.push_back(MyFlags);
11818 CLI.OutVals.push_back(Parts[j]);
11819 }
11820
11821 if (NeedsRegBlock && Value == NumValues - 1)
11822 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11823 }
11824 }
11825
11827 CLI.Chain = LowerCall(CLI, InVals);
11828
11829 // Update CLI.InVals to use outside of this function.
11830 CLI.InVals = InVals;
11831
11832 // Verify that the target's LowerCall behaved as expected.
11833 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11834 "LowerCall didn't return a valid chain!");
11835 assert((!CLI.IsTailCall || InVals.empty()) &&
11836 "LowerCall emitted a return value for a tail call!");
11837 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11838 "LowerCall didn't emit the correct number of values!");
11839
11840 // For a tail call, the return value is merely live-out and there aren't
11841 // any nodes in the DAG representing it. Return a special value to
11842 // indicate that a tail call has been emitted and no more Instructions
11843 // should be processed in the current block.
11844 if (CLI.IsTailCall) {
11845 CLI.DAG.setRoot(CLI.Chain);
11846 return std::make_pair(SDValue(), SDValue());
11847 }
11848
11849#ifndef NDEBUG
11850 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11851 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11852 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11853 "LowerCall emitted a value with the wrong type!");
11854 }
11855#endif
11856
11857 SmallVector<SDValue, 4> ReturnValues;
11858 if (!CanLowerReturn) {
11859 // The instruction result is the result of loading from the
11860 // hidden sret parameter.
11861 MVT PtrVT = getPointerTy(DL, DL.getAllocaAddrSpace());
11862
11863 unsigned NumValues = RetVTs.size();
11864 ReturnValues.resize(NumValues);
11865 SmallVector<SDValue, 4> Chains(NumValues);
11866
11867 // An aggregate return value cannot wrap around the address space, so
11868 // offsets to its parts don't wrap either.
11870 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
11871 for (unsigned i = 0; i < NumValues; ++i) {
11873 DemoteStackSlot, CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT),
11875 SDValue L = CLI.DAG.getLoad(
11876 RetVTs[i], CLI.DL, CLI.Chain, Add,
11878 DemoteStackIdx, Offsets[i]),
11879 HiddenSRetAlign);
11880 ReturnValues[i] = L;
11881 Chains[i] = L.getValue(1);
11882 }
11883
11884 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
11885 } else {
11886 // Collect the legal value parts into potentially illegal values
11887 // that correspond to the original function's return values.
11888 std::optional<ISD::NodeType> AssertOp;
11889 if (CLI.RetSExt)
11890 AssertOp = ISD::AssertSext;
11891 else if (CLI.RetZExt)
11892 AssertOp = ISD::AssertZext;
11893 unsigned CurReg = 0;
11894 for (EVT VT : RetVTs) {
11895 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11896 unsigned NumRegs =
11897 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11898
11899 ReturnValues.push_back(getCopyFromParts(
11900 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
11901 CLI.Chain, CLI.CallConv, AssertOp));
11902 CurReg += NumRegs;
11903 }
11904
11905 // For a function returning void, there is no return value. We can't create
11906 // such a node, so we just return a null return value in that case. In
11907 // that case, nothing will actually look at the value.
11908 if (ReturnValues.empty())
11909 return std::make_pair(SDValue(), CLI.Chain);
11910 }
11911
11912 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
11913 CLI.DAG.getVTList(RetVTs), ReturnValues);
11914 return std::make_pair(Res, CLI.Chain);
11915}
11916
11917/// Places new result values for the node in Results (their number
11918/// and types must exactly match those of the original return values of
11919/// the node), or leaves Results empty, which indicates that the node is not
11920/// to be custom lowered after all.
11923 SelectionDAG &DAG) const {
11924 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
11925
11926 if (!Res.getNode())
11927 return;
11928
11929 // If the original node has one result, take the return value from
11930 // LowerOperation as is. It might not be result number 0.
11931 if (N->getNumValues() == 1) {
11932 Results.push_back(Res);
11933 return;
11934 }
11935
11936 // If the original node has multiple results, then the return node should
11937 // have the same number of results.
11938 assert((N->getNumValues() == Res->getNumValues()) &&
11939 "Lowering returned the wrong number of results!");
11940
11941 // Places new result values base on N result number.
11942 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11943 Results.push_back(Res.getValue(I));
11944}
11945
11947 llvm_unreachable("LowerOperation not implemented for this target!");
11948}
11949
11951 Register Reg,
11952 ISD::NodeType ExtendType) {
11954 assert((Op.getOpcode() != ISD::CopyFromReg ||
11955 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11956 "Copy from a reg to the same reg!");
11957 assert(!Reg.isPhysical() && "Is a physreg");
11958
11959 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11960 // If this is an InlineAsm we have to match the registers required, not the
11961 // notional registers required by the type.
11962
11963 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11964 std::nullopt); // This is not an ABI copy.
11965 SDValue Chain = DAG.getEntryNode();
11966
11967 if (ExtendType == ISD::ANY_EXTEND) {
11968 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
11969 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11970 ExtendType = PreferredExtendIt->second;
11971 }
11972 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
11973 PendingExports.push_back(Chain);
11974}
11975
11977
11978/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11979/// entry block, return true. This includes arguments used by switches, since
11980/// the switch may expand into multiple basic blocks.
11981static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11982 // With FastISel active, we may be splitting blocks, so force creation
11983 // of virtual registers for all non-dead arguments.
11984 if (FastISel)
11985 return A->use_empty();
11986
11987 const BasicBlock &Entry = A->getParent()->front();
11988 for (const User *U : A->users())
11989 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
11990 return false; // Use not in entry block.
11991
11992 return true;
11993}
11994
11996 DenseMap<const Argument *,
11997 std::pair<const AllocaInst *, const StoreInst *>>;
11998
11999/// Scan the entry block of the function in FuncInfo for arguments that look
12000/// like copies into a local alloca. Record any copied arguments in
12001/// ArgCopyElisionCandidates.
12002static void
12004 FunctionLoweringInfo *FuncInfo,
12005 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
12006 // Record the state of every static alloca used in the entry block. Argument
12007 // allocas are all used in the entry block, so we need approximately as many
12008 // entries as we have arguments.
12009 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
12011 unsigned NumArgs = FuncInfo->Fn->arg_size();
12012 StaticAllocas.reserve(NumArgs * 2);
12013
12014 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
12015 if (!V)
12016 return nullptr;
12017 V = V->stripPointerCasts();
12018 const auto *AI = dyn_cast<AllocaInst>(V);
12019 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
12020 return nullptr;
12021 auto Iter = StaticAllocas.insert({AI, Unknown});
12022 return &Iter.first->second;
12023 };
12024
12025 // Look for stores of arguments to static allocas. Look through bitcasts and
12026 // GEPs to handle type coercions, as long as the alloca is fully initialized
12027 // by the store. Any non-store use of an alloca escapes it and any subsequent
12028 // unanalyzed store might write it.
12029 // FIXME: Handle structs initialized with multiple stores.
12030 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
12031 // Look for stores, and handle non-store uses conservatively.
12032 const auto *SI = dyn_cast<StoreInst>(&I);
12033 if (!SI) {
12034 // We will look through cast uses, so ignore them completely.
12035 if (I.isCast())
12036 continue;
12037 // Ignore pseudo op intrinsics, they don't escape or store to allocas.
12039 continue;
12040 // This is an unknown instruction. Assume it escapes or writes to all
12041 // static alloca operands.
12042 for (const Use &U : I.operands()) {
12043 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
12044 *Info = StaticAllocaInfo::Clobbered;
12045 }
12046 continue;
12047 }
12048
12049 // If the stored value is a static alloca, mark it as escaped.
12050 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
12051 *Info = StaticAllocaInfo::Clobbered;
12052
12053 // Check if the destination is a static alloca.
12054 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
12055 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
12056 if (!Info)
12057 continue;
12058 const AllocaInst *AI = cast<AllocaInst>(Dst);
12059
12060 // Skip allocas that have been initialized or clobbered.
12061 if (*Info != StaticAllocaInfo::Unknown)
12062 continue;
12063
12064 // Check if the stored value is an argument, and that this store fully
12065 // initializes the alloca.
12066 // If the argument type has padding bits we can't directly forward a pointer
12067 // as the upper bits may contain garbage.
12068 // Don't elide copies from the same argument twice.
12069 const Value *Val = SI->getValueOperand()->stripPointerCasts();
12070 const auto *Arg = dyn_cast<Argument>(Val);
12071 std::optional<TypeSize> AllocaSize = AI->getAllocationSize(DL);
12072 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
12073 Arg->getType()->isEmptyTy() || !AllocaSize ||
12074 DL.getTypeStoreSize(Arg->getType()) != *AllocaSize ||
12075 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
12076 ArgCopyElisionCandidates.count(Arg)) {
12077 *Info = StaticAllocaInfo::Clobbered;
12078 continue;
12079 }
12080
12081 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
12082 << '\n');
12083
12084 // Mark this alloca and store for argument copy elision.
12085 *Info = StaticAllocaInfo::Elidable;
12086 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
12087
12088 // Stop scanning if we've seen all arguments. This will happen early in -O0
12089 // builds, which is useful, because -O0 builds have large entry blocks and
12090 // many allocas.
12091 if (ArgCopyElisionCandidates.size() == NumArgs)
12092 break;
12093 }
12094}
12095
12096/// Try to elide argument copies from memory into a local alloca. Succeeds if
12097/// ArgVal is a load from a suitable fixed stack object.
12100 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
12101 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
12102 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
12103 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
12104 // Check if this is a load from a fixed stack object.
12105 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
12106 if (!LNode)
12107 return;
12108 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
12109 if (!FINode)
12110 return;
12111
12112 // Check that the fixed stack object is the right size and alignment.
12113 // Look at the alignment that the user wrote on the alloca instead of looking
12114 // at the stack object.
12115 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
12116 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
12117 const AllocaInst *AI = ArgCopyIter->second.first;
12118 int FixedIndex = FINode->getIndex();
12119 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
12120 int OldIndex = AllocaIndex;
12121 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
12122 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
12123 LLVM_DEBUG(
12124 dbgs() << " argument copy elision failed due to bad fixed stack "
12125 "object size\n");
12126 return;
12127 }
12128 Align RequiredAlignment = AI->getAlign();
12129 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
12130 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
12131 "greater than stack argument alignment ("
12132 << DebugStr(RequiredAlignment) << " vs "
12133 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
12134 return;
12135 }
12136
12137 // Perform the elision. Delete the old stack object and replace its only use
12138 // in the variable info map. Mark the stack object as mutable and aliased.
12139 LLVM_DEBUG({
12140 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
12141 << " Replacing frame index " << OldIndex << " with " << FixedIndex
12142 << '\n';
12143 });
12144 MFI.RemoveStackObject(OldIndex);
12145 MFI.setIsImmutableObjectIndex(FixedIndex, false);
12146 MFI.setIsAliasedObjectIndex(FixedIndex, true);
12147 AllocaIndex = FixedIndex;
12148 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
12149 for (SDValue ArgVal : ArgVals)
12150 Chains.push_back(ArgVal.getValue(1));
12151
12152 // Avoid emitting code for the store implementing the copy.
12153 const StoreInst *SI = ArgCopyIter->second.second;
12154 ElidedArgCopyInstrs.insert(SI);
12155
12156 // Check for uses of the argument again so that we can avoid exporting ArgVal
12157 // if it is't used by anything other than the store.
12158 for (const Value *U : Arg.users()) {
12159 if (U != SI) {
12160 ArgHasUses = true;
12161 break;
12162 }
12163 }
12164}
12165
12166void SelectionDAGISel::LowerArguments(const Function &F, bool FastISel) {
12167 SelectionDAG &DAG = SDB->DAG;
12168 SDLoc dl = SDB->getCurSDLoc();
12169 const DataLayout &DL = DAG.getDataLayout();
12171
12172 // In Naked functions we aren't going to save any registers.
12173 if (F.hasFnAttribute(Attribute::Naked))
12174 return;
12175
12176 if (!FuncInfo->CanLowerReturn) {
12177 // Put in an sret pointer parameter before all the other parameters.
12178 MVT ValueVT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
12179
12180 ISD::ArgFlagsTy Flags;
12181 Flags.setSRet();
12182 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVT);
12183 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
12185 Ins.push_back(RetArg);
12186 }
12187
12188 // Look for stores of arguments to static allocas. Mark such arguments with a
12189 // flag to ask the target to give us the memory location of that argument if
12190 // available.
12191 ArgCopyElisionMapTy ArgCopyElisionCandidates;
12193 ArgCopyElisionCandidates);
12194
12195 // Set up the incoming argument description vector.
12196 for (const Argument &Arg : F.args()) {
12197 unsigned ArgNo = Arg.getArgNo();
12199 ComputeValueTypes(DAG.getDataLayout(), Arg.getType(), Types);
12200 bool isArgValueUsed = !Arg.use_empty();
12201 Type *FinalType = Arg.getType();
12202 if (Arg.hasAttribute(Attribute::ByVal))
12203 FinalType = Arg.getParamByValType();
12204 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
12205 FinalType, F.getCallingConv(), F.isVarArg(), DL);
12206 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
12207 ++Value) {
12208 Type *ArgTy = Types[Value];
12209 EVT VT = TLI->getValueType(DL, ArgTy);
12210 ISD::ArgFlagsTy Flags;
12211
12212 if (ArgTy->isPointerTy()) {
12213 Flags.setPointer();
12214 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
12215 }
12216 if (Arg.hasAttribute(Attribute::ZExt))
12217 Flags.setZExt();
12218 if (Arg.hasAttribute(Attribute::SExt))
12219 Flags.setSExt();
12220 if (Arg.hasAttribute(Attribute::InReg)) {
12221 // If we are using vectorcall calling convention, a structure that is
12222 // passed InReg - is surely an HVA
12223 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
12224 isa<StructType>(Arg.getType())) {
12225 // The first value of a structure is marked
12226 if (0 == Value)
12227 Flags.setHvaStart();
12228 Flags.setHva();
12229 }
12230 // Set InReg Flag
12231 Flags.setInReg();
12232 }
12233 if (Arg.hasAttribute(Attribute::StructRet))
12234 Flags.setSRet();
12235 if (Arg.hasAttribute(Attribute::SwiftSelf))
12236 Flags.setSwiftSelf();
12237 if (Arg.hasAttribute(Attribute::SwiftAsync))
12238 Flags.setSwiftAsync();
12239 if (Arg.hasAttribute(Attribute::SwiftError))
12240 Flags.setSwiftError();
12241 if (Arg.hasAttribute(Attribute::ByVal))
12242 Flags.setByVal();
12243 if (Arg.hasAttribute(Attribute::ByRef))
12244 Flags.setByRef();
12245 if (Arg.hasAttribute(Attribute::InAlloca)) {
12246 Flags.setInAlloca();
12247 // Set the byval flag for CCAssignFn callbacks that don't know about
12248 // inalloca. This way we can know how many bytes we should've allocated
12249 // and how many bytes a callee cleanup function will pop. If we port
12250 // inalloca to more targets, we'll have to add custom inalloca handling
12251 // in the various CC lowering callbacks.
12252 Flags.setByVal();
12253 }
12254 if (Arg.hasAttribute(Attribute::Preallocated)) {
12255 Flags.setPreallocated();
12256 // Set the byval flag for CCAssignFn callbacks that don't know about
12257 // preallocated. This way we can know how many bytes we should've
12258 // allocated and how many bytes a callee cleanup function will pop. If
12259 // we port preallocated to more targets, we'll have to add custom
12260 // preallocated handling in the various CC lowering callbacks.
12261 Flags.setByVal();
12262 }
12263
12264 // Certain targets (such as MIPS), may have a different ABI alignment
12265 // for a type depending on the context. Give the target a chance to
12266 // specify the alignment it wants.
12267 const Align OriginalAlignment(
12268 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
12269 Flags.setOrigAlign(OriginalAlignment);
12270
12271 Align MemAlign;
12272 Type *ArgMemTy = nullptr;
12273 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
12274 Flags.isByRef()) {
12275 if (!ArgMemTy)
12276 ArgMemTy = Arg.getPointeeInMemoryValueType();
12277
12278 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
12279
12280 // For in-memory arguments, size and alignment should be passed from FE.
12281 // BE will guess if this info is not there but there are cases it cannot
12282 // get right.
12283 if (auto ParamAlign = Arg.getParamStackAlign())
12284 MemAlign = *ParamAlign;
12285 else if ((ParamAlign = Arg.getParamAlign()))
12286 MemAlign = *ParamAlign;
12287 else
12288 MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL);
12289 if (Flags.isByRef())
12290 Flags.setByRefSize(MemSize);
12291 else
12292 Flags.setByValSize(MemSize);
12293 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
12294 MemAlign = *ParamAlign;
12295 } else {
12296 MemAlign = OriginalAlignment;
12297 }
12298 Flags.setMemAlign(MemAlign);
12299
12300 if (Arg.hasAttribute(Attribute::Nest))
12301 Flags.setNest();
12302 if (NeedsRegBlock)
12303 Flags.setInConsecutiveRegs();
12304 if (ArgCopyElisionCandidates.count(&Arg))
12305 Flags.setCopyElisionCandidate();
12306 if (Arg.hasAttribute(Attribute::Returned))
12307 Flags.setReturned();
12308
12309 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
12310 *CurDAG->getContext(), F.getCallingConv(), VT);
12311 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
12312 *CurDAG->getContext(), F.getCallingConv(), VT);
12313 for (unsigned i = 0; i != NumRegs; ++i) {
12314 // For scalable vectors, use the minimum size; individual targets
12315 // are responsible for handling scalable vector arguments and
12316 // return values.
12317 ISD::InputArg MyFlags(
12318 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12319 i * RegisterVT.getStoreSize().getKnownMinValue());
12320 if (NumRegs > 1 && i == 0)
12321 MyFlags.Flags.setSplit();
12322 // if it isn't first piece, alignment must be 1
12323 else if (i > 0) {
12324 MyFlags.Flags.setOrigAlign(Align(1));
12325 if (i == NumRegs - 1)
12326 MyFlags.Flags.setSplitEnd();
12327 }
12328 Ins.push_back(MyFlags);
12329 }
12330 if (NeedsRegBlock && Value == NumValues - 1)
12331 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
12332 }
12333 }
12334
12335 // Call the target to set up the argument values.
12337 SDValue NewRoot = TLI->LowerFormalArguments(
12338 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
12339
12340 // Verify that the target's LowerFormalArguments behaved as expected.
12341 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
12342 "LowerFormalArguments didn't return a valid chain!");
12343 assert(InVals.size() == Ins.size() &&
12344 "LowerFormalArguments didn't emit the correct number of values!");
12345 assert(all_of(InVals, [](SDValue InVal) { return InVal.getNode(); }) &&
12346 "LowerFormalArguments emitted a null value!");
12347
12348 // Update the DAG with the new chain value resulting from argument lowering.
12349 DAG.setRoot(NewRoot);
12350
12351 // Set up the argument values.
12352 unsigned i = 0;
12353 if (!FuncInfo->CanLowerReturn) {
12354 // Create a virtual register for the sret pointer, and put in a copy
12355 // from the sret argument into it.
12356 MVT VT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
12357 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
12358 std::optional<ISD::NodeType> AssertOp;
12359 SDValue ArgValue =
12360 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
12361 F.getCallingConv(), AssertOp);
12362
12363 MachineFunction& MF = SDB->DAG.getMachineFunction();
12364 MachineRegisterInfo& RegInfo = MF.getRegInfo();
12365 Register SRetReg =
12366 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
12367 FuncInfo->DemoteRegister = SRetReg;
12368 NewRoot =
12369 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
12370 DAG.setRoot(NewRoot);
12371
12372 // i indexes lowered arguments. Bump it past the hidden sret argument.
12373 ++i;
12374 }
12375
12377 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12378 for (const Argument &Arg : F.args()) {
12379 SmallVector<SDValue, 4> ArgValues;
12380 SmallVector<EVT, 4> ValueVTs;
12381 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
12382 unsigned NumValues = ValueVTs.size();
12383 if (NumValues == 0)
12384 continue;
12385
12386 bool ArgHasUses = !Arg.use_empty();
12387
12388 // Elide the copying store if the target loaded this argument from a
12389 // suitable fixed stack object.
12390 if (Ins[i].Flags.isCopyElisionCandidate()) {
12391 unsigned NumParts = 0;
12392 for (EVT VT : ValueVTs)
12393 NumParts += TLI->getNumRegistersForCallingConv(*CurDAG->getContext(),
12394 F.getCallingConv(), VT);
12395
12396 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
12397 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
12398 ArrayRef(&InVals[i], NumParts), ArgHasUses);
12399 }
12400
12401 // If this argument is unused then remember its value. It is used to generate
12402 // debugging information.
12403 bool isSwiftErrorArg =
12404 TLI->supportSwiftError() &&
12405 Arg.hasAttribute(Attribute::SwiftError);
12406 if (!ArgHasUses && !isSwiftErrorArg) {
12407 SDB->setUnusedArgValue(&Arg, InVals[i]);
12408
12409 // Also remember any frame index for use in FastISel.
12410 if (FrameIndexSDNode *FI =
12412 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12413 }
12414
12415 for (unsigned Val = 0; Val != NumValues; ++Val) {
12416 EVT VT = ValueVTs[Val];
12417 MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
12418 F.getCallingConv(), VT);
12419 unsigned NumParts = TLI->getNumRegistersForCallingConv(
12420 *CurDAG->getContext(), F.getCallingConv(), VT);
12421
12422 // Even an apparent 'unused' swifterror argument needs to be returned. So
12423 // we do generate a copy for it that can be used on return from the
12424 // function.
12425 if (ArgHasUses || isSwiftErrorArg) {
12426 std::optional<ISD::NodeType> AssertOp;
12427 if (Arg.hasAttribute(Attribute::SExt))
12428 AssertOp = ISD::AssertSext;
12429 else if (Arg.hasAttribute(Attribute::ZExt))
12430 AssertOp = ISD::AssertZext;
12431
12432 SDValue OutVal =
12433 getCopyFromParts(DAG, dl, &InVals[i], NumParts, PartVT, VT, nullptr,
12434 NewRoot, F.getCallingConv(), AssertOp);
12435
12436 FPClassTest NoFPClass = Arg.getNoFPClass();
12437 if (NoFPClass != fcNone) {
12438 SDValue SDNoFPClass = DAG.getTargetConstant(
12439 static_cast<uint64_t>(NoFPClass), dl, MVT::i32);
12440 OutVal = DAG.getNode(ISD::AssertNoFPClass, dl, OutVal.getValueType(),
12441 OutVal, SDNoFPClass);
12442 }
12443 ArgValues.push_back(OutVal);
12444 }
12445
12446 i += NumParts;
12447 }
12448
12449 // We don't need to do anything else for unused arguments.
12450 if (ArgValues.empty())
12451 continue;
12452
12453 // Note down frame index.
12454 if (FrameIndexSDNode *FI =
12455 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
12456 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12457
12458 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
12459 SDB->getCurSDLoc());
12460
12461 SDB->setValue(&Arg, Res);
12462 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
12463 // We want to associate the argument with the frame index, among
12464 // involved operands, that correspond to the lowest address. The
12465 // getCopyFromParts function, called earlier, is swapping the order of
12466 // the operands to BUILD_PAIR depending on endianness. The result of
12467 // that swapping is that the least significant bits of the argument will
12468 // be in the first operand of the BUILD_PAIR node, and the most
12469 // significant bits will be in the second operand.
12470 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
12471 if (LoadSDNode *LNode =
12472 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
12473 if (FrameIndexSDNode *FI =
12474 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
12475 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12476 }
12477
12478 // Analyses past this point are naive and don't expect an assertion.
12479 if (Res.getOpcode() == ISD::AssertZext)
12480 Res = Res.getOperand(0);
12481
12482 // Update the SwiftErrorVRegDefMap.
12483 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
12484 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
12485 if (Reg.isVirtual())
12486 SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
12487 Reg);
12488 }
12489
12490 // If this argument is live outside of the entry block, insert a copy from
12491 // wherever we got it to the vreg that other BB's will reference it as.
12492 if (Res.getOpcode() == ISD::CopyFromReg) {
12493 // If we can, though, try to skip creating an unnecessary vreg.
12494 // FIXME: This isn't very clean... it would be nice to make this more
12495 // general.
12496 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
12497 if (Reg.isVirtual()) {
12498 FuncInfo->ValueMap[&Arg] = Reg;
12499 continue;
12500 }
12501 }
12502 if (!isOnlyUsedInEntryBlock(&Arg, FastISel)) {
12503 FuncInfo->InitializeRegForValue(&Arg);
12504 SDB->CopyToExportRegsIfNeeded(&Arg);
12505 }
12506 }
12507
12508 if (!Chains.empty()) {
12509 Chains.push_back(NewRoot);
12510 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
12511 }
12512
12513 DAG.setRoot(NewRoot);
12514
12515 assert(i == InVals.size() && "Argument register count mismatch!");
12516
12517 // If any argument copy elisions occurred and we have debug info, update the
12518 // stale frame indices used in the dbg.declare variable info table.
12519 if (!ArgCopyElisionFrameIndexMap.empty()) {
12520 for (MachineFunction::VariableDbgInfo &VI :
12521 MF->getInStackSlotVariableDbgInfo()) {
12522 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
12523 if (I != ArgCopyElisionFrameIndexMap.end())
12524 VI.updateStackSlot(I->second);
12525 }
12526 }
12527
12528 // Finally, if the target has anything special to do, allow it to do so.
12530}
12531
12532/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
12533/// ensure constants are generated when needed. Remember the virtual registers
12534/// that need to be added to the Machine PHI nodes as input. We cannot just
12535/// directly add them, because expansion might result in multiple MBB's for one
12536/// BB. As such, the start of the BB might correspond to a different MBB than
12537/// the end.
12538void
12539SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
12540 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12541
12542 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12543
12544 // Check PHI nodes in successors that expect a value to be available from this
12545 // block.
12546 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
12547 if (!isa<PHINode>(SuccBB->begin())) continue;
12548 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB);
12549
12550 // If this terminator has multiple identical successors (common for
12551 // switches), only handle each succ once.
12552 if (!SuccsHandled.insert(SuccMBB).second)
12553 continue;
12554
12556
12557 // At this point we know that there is a 1-1 correspondence between LLVM PHI
12558 // nodes and Machine PHI nodes, but the incoming operands have not been
12559 // emitted yet.
12560 for (const PHINode &PN : SuccBB->phis()) {
12561 // Ignore dead phi's.
12562 if (PN.use_empty())
12563 continue;
12564
12565 // Skip empty types
12566 if (PN.getType()->isEmptyTy())
12567 continue;
12568
12569 Register Reg;
12570 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
12571
12572 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
12573 Register &RegOut = ConstantsOut[C];
12574 if (!RegOut) {
12575 RegOut = FuncInfo.CreateRegs(PHIOp);
12576 // We need to zero/sign extend ConstantInt phi operands to match
12577 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12578 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12579 if (auto *CI = dyn_cast<ConstantInt>(C))
12580 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
12582 CopyValueToVirtualRegister(C, RegOut, ExtendType);
12583 }
12584 Reg = RegOut;
12585 } else {
12586 auto I = FuncInfo.ValueMap.find(PHIOp);
12587 if (I != FuncInfo.ValueMap.end())
12588 Reg = I->second;
12589 else {
12590 assert(isa<AllocaInst>(PHIOp) &&
12591 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12592 "Didn't codegen value into a register!??");
12593 Reg = FuncInfo.CreateRegs(PHIOp);
12595 }
12596 }
12597
12598 // Remember that this register needs to added to the machine PHI node as
12599 // the input for this MBB.
12600 SmallVector<EVT, 4> ValueVTs;
12601 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
12602 for (EVT VT : ValueVTs) {
12603 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
12604 for (unsigned i = 0; i != NumRegisters; ++i)
12605 FuncInfo.PHINodesToUpdate.emplace_back(&*MBBI++, Reg + i);
12606 Reg += NumRegisters;
12607 }
12608 }
12609 }
12610
12611 ConstantsOut.clear();
12612}
12613
12614MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12616 if (++I == FuncInfo.MF->end())
12617 return nullptr;
12618 return &*I;
12619}
12620
12621/// During lowering new call nodes can be created (such as memset, etc.).
12622/// Those will become new roots of the current DAG, but complications arise
12623/// when they are tail calls. In such cases, the call lowering will update
12624/// the root, but the builder still needs to know that a tail call has been
12625/// lowered in order to avoid generating an additional return.
12626void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12627 // If the node is null, we do have a tail call.
12628 if (MaybeTC.getNode() != nullptr)
12629 DAG.setRoot(MaybeTC);
12630 else
12631 HasTailCall = true;
12632}
12633
12634void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12635 MachineBasicBlock *SwitchMBB,
12636 MachineBasicBlock *DefaultMBB) {
12637 MachineFunction *CurMF = FuncInfo.MF;
12638 MachineBasicBlock *NextMBB = nullptr;
12640 if (++BBI != FuncInfo.MF->end())
12641 NextMBB = &*BBI;
12642
12643 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12644
12645 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12646
12647 if (Size == 2 && W.MBB == SwitchMBB) {
12648 // If any two of the cases has the same destination, and if one value
12649 // is the same as the other, but has one bit unset that the other has set,
12650 // use bit manipulation to do two compares at once. For example:
12651 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12652 // TODO: This could be extended to merge any 2 cases in switches with 3
12653 // cases.
12654 // TODO: Handle cases where W.CaseBB != SwitchBB.
12655 CaseCluster &Small = *W.FirstCluster;
12656 CaseCluster &Big = *W.LastCluster;
12657
12658 if (Small.Low == Small.High && Big.Low == Big.High &&
12659 Small.MBB == Big.MBB) {
12660 const APInt &SmallValue = Small.Low->getValue();
12661 const APInt &BigValue = Big.Low->getValue();
12662
12663 // Check that there is only one bit different.
12664 APInt CommonBit = BigValue ^ SmallValue;
12665 if (CommonBit.isPowerOf2()) {
12666 SDValue CondLHS = getValue(Cond);
12667 EVT VT = CondLHS.getValueType();
12668 SDLoc DL = getCurSDLoc();
12669
12670 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
12671 DAG.getConstant(CommonBit, DL, VT));
12672 SDValue Cond = DAG.getSetCC(
12673 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
12674 ISD::SETEQ);
12675
12676 // Update successor info.
12677 // Both Small and Big will jump to Small.BB, so we sum up the
12678 // probabilities.
12679 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
12680 if (BPI)
12681 addSuccessorWithProb(
12682 SwitchMBB, DefaultMBB,
12683 // The default destination is the first successor in IR.
12684 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
12685 else
12686 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12687
12688 // Insert the true branch.
12689 SDValue BrCond =
12690 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
12691 DAG.getBasicBlock(Small.MBB));
12692 // Insert the false branch.
12693 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
12694 DAG.getBasicBlock(DefaultMBB));
12695
12696 DAG.setRoot(BrCond);
12697 return;
12698 }
12699 }
12700 }
12701
12702 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12703 // Here, we order cases by probability so the most likely case will be
12704 // checked first. However, two clusters can have the same probability in
12705 // which case their relative ordering is non-deterministic. So we use Low
12706 // as a tie-breaker as clusters are guaranteed to never overlap.
12707 llvm::sort(W.FirstCluster, W.LastCluster + 1,
12708 [](const CaseCluster &a, const CaseCluster &b) {
12709 return a.Prob != b.Prob ?
12710 a.Prob > b.Prob :
12711 a.Low->getValue().slt(b.Low->getValue());
12712 });
12713
12714 // Rearrange the case blocks so that the last one falls through if possible
12715 // without changing the order of probabilities.
12716 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12717 --I;
12718 if (I->Prob > W.LastCluster->Prob)
12719 break;
12720 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12721 std::swap(*I, *W.LastCluster);
12722 break;
12723 }
12724 }
12725 }
12726
12727 // Compute total probability.
12728 BranchProbability DefaultProb = W.DefaultProb;
12729 BranchProbability UnhandledProbs = DefaultProb;
12730 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12731 UnhandledProbs += I->Prob;
12732
12733 MachineBasicBlock *CurMBB = W.MBB;
12734 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12735 bool FallthroughUnreachable = false;
12736 MachineBasicBlock *Fallthrough;
12737 if (I == W.LastCluster) {
12738 // For the last cluster, fall through to the default destination.
12739 Fallthrough = DefaultMBB;
12740 FallthroughUnreachable =
12742 } else {
12743 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
12744 CurMF->insert(BBI, Fallthrough);
12745 // Put Cond in a virtual register to make it available from the new blocks.
12747 }
12748 UnhandledProbs -= I->Prob;
12749
12750 switch (I->Kind) {
12751 case CC_JumpTable: {
12752 // FIXME: Optimize away range check based on pivot comparisons.
12753 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12754 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12755
12756 // The jump block hasn't been inserted yet; insert it here.
12757 MachineBasicBlock *JumpMBB = JT->MBB;
12758 CurMF->insert(BBI, JumpMBB);
12759
12760 auto JumpProb = I->Prob;
12761 auto FallthroughProb = UnhandledProbs;
12762
12763 // If the default statement is a target of the jump table, we evenly
12764 // distribute the default probability to successors of CurMBB. Also
12765 // update the probability on the edge from JumpMBB to Fallthrough.
12766 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12767 SE = JumpMBB->succ_end();
12768 SI != SE; ++SI) {
12769 if (*SI == DefaultMBB) {
12770 JumpProb += DefaultProb / 2;
12771 FallthroughProb -= DefaultProb / 2;
12772 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
12773 JumpMBB->normalizeSuccProbs();
12774 break;
12775 }
12776 }
12777
12778 // If the default clause is unreachable, propagate that knowledge into
12779 // JTH->FallthroughUnreachable which will use it to suppress the range
12780 // check.
12781 //
12782 // However, don't do this if we're doing branch target enforcement
12783 // (e.g. Arm BTI, or the cf-protection-branch module flag used by
12784 // RISC-V Zicfilp and X86 IBT), because a table branch _without_ a
12785 // range check can be a tempting JOP gadget - out-of-bounds inputs
12786 // that are impossible in correct execution become possible again if
12787 // an attacker can influence the control flow. So if an attacker
12788 // doesn't already have a branch target enforcement bypass available,
12789 // we don't want them to be able to get one out of this table branch.
12790 if (FallthroughUnreachable) {
12791 Function &CurFunc = CurMF->getFunction();
12792 bool HasCFBranch = false;
12793 if (auto *CFBranch = mdconst::extract_or_null<ConstantInt>(
12794 CurFunc.getParent()->getModuleFlag("cf-protection-branch")))
12795 HasCFBranch = !CFBranch->isZero();
12796 if (!HasCFBranch &&
12797 !CurFunc.hasFnAttribute("branch-target-enforcement"))
12798 JTH->FallthroughUnreachable = true;
12799 }
12800
12801 if (!JTH->FallthroughUnreachable)
12802 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12803 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12804 CurMBB->normalizeSuccProbs();
12805
12806 // The jump table header will be inserted in our current block, do the
12807 // range check, and fall through to our fallthrough block.
12808 JTH->HeaderBB = CurMBB;
12809 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12810
12811 // If we're in the right place, emit the jump table header right now.
12812 if (CurMBB == SwitchMBB) {
12813 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
12814 JTH->Emitted = true;
12815 }
12816 break;
12817 }
12818 case CC_BitTests: {
12819 // FIXME: Optimize away range check based on pivot comparisons.
12820 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12821
12822 // The bit test blocks haven't been inserted yet; insert them here.
12823 for (BitTestCase &BTC : BTB->Cases)
12824 CurMF->insert(BBI, BTC.ThisBB);
12825
12826 // Fill in fields of the BitTestBlock.
12827 BTB->Parent = CurMBB;
12828 BTB->Default = Fallthrough;
12829
12830 BTB->DefaultProb = UnhandledProbs;
12831 // If the cases in bit test don't form a contiguous range, we evenly
12832 // distribute the probability on the edge to Fallthrough to two
12833 // successors of CurMBB.
12834 if (!BTB->ContiguousRange) {
12835 BTB->Prob += DefaultProb / 2;
12836 BTB->DefaultProb -= DefaultProb / 2;
12837 }
12838
12839 if (FallthroughUnreachable)
12840 BTB->FallthroughUnreachable = true;
12841
12842 // If we're in the right place, emit the bit test header right now.
12843 if (CurMBB == SwitchMBB) {
12844 visitBitTestHeader(*BTB, SwitchMBB);
12845 BTB->Emitted = true;
12846 }
12847 break;
12848 }
12849 case CC_Range: {
12850 const Value *RHS, *LHS, *MHS;
12852 if (I->Low == I->High) {
12853 // Check Cond == I->Low.
12854 CC = ISD::SETEQ;
12855 LHS = Cond;
12856 RHS=I->Low;
12857 MHS = nullptr;
12858 } else {
12859 // Check I->Low <= Cond <= I->High.
12860 CC = ISD::SETLE;
12861 LHS = I->Low;
12862 MHS = Cond;
12863 RHS = I->High;
12864 }
12865
12866 // If Fallthrough is unreachable, fold away the comparison.
12867 if (FallthroughUnreachable)
12868 CC = ISD::SETTRUE;
12869
12870 // The false probability is the sum of all unhandled cases.
12871 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12872 getCurSDLoc(), I->Prob, UnhandledProbs);
12873
12874 if (CurMBB == SwitchMBB)
12875 visitSwitchCase(CB, SwitchMBB);
12876 else
12877 SL->SwitchCases.push_back(CB);
12878
12879 break;
12880 }
12881 }
12882 CurMBB = Fallthrough;
12883 }
12884}
12885
12886void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12887 const SwitchWorkListItem &W,
12888 Value *Cond,
12889 MachineBasicBlock *SwitchMBB) {
12890 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12891 "Clusters not sorted?");
12892 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12893
12894 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12895 SL->computeSplitWorkItemInfo(W);
12896
12897 // Use the first element on the right as pivot since we will make less-than
12898 // comparisons against it.
12899 CaseClusterIt PivotCluster = FirstRight;
12900 assert(PivotCluster > W.FirstCluster);
12901 assert(PivotCluster <= W.LastCluster);
12902
12903 CaseClusterIt FirstLeft = W.FirstCluster;
12904 CaseClusterIt LastRight = W.LastCluster;
12905
12906 const ConstantInt *Pivot = PivotCluster->Low;
12907
12908 // New blocks will be inserted immediately after the current one.
12910 ++BBI;
12911
12912 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12913 // we can branch to its destination directly if it's squeezed exactly in
12914 // between the known lower bound and Pivot - 1.
12915 MachineBasicBlock *LeftMBB;
12916 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12917 FirstLeft->Low == W.GE &&
12918 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12919 LeftMBB = FirstLeft->MBB;
12920 } else {
12921 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12922 FuncInfo.MF->insert(BBI, LeftMBB);
12923 WorkList.push_back(
12924 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
12925 // Put Cond in a virtual register to make it available from the new blocks.
12927 }
12928
12929 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12930 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12931 // directly if RHS.High equals the current upper bound.
12932 MachineBasicBlock *RightMBB;
12933 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12934 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12935 RightMBB = FirstRight->MBB;
12936 } else {
12937 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12938 FuncInfo.MF->insert(BBI, RightMBB);
12939 WorkList.push_back(
12940 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
12941 // Put Cond in a virtual register to make it available from the new blocks.
12943 }
12944
12945 // Create the CaseBlock record that will be used to lower the branch.
12946 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12947 getCurSDLoc(), LeftProb, RightProb);
12948
12949 if (W.MBB == SwitchMBB)
12950 visitSwitchCase(CB, SwitchMBB);
12951 else
12952 SL->SwitchCases.push_back(CB);
12953}
12954
12955// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12956// from the swith statement.
12958 BranchProbability PeeledCaseProb) {
12959 if (PeeledCaseProb == BranchProbability::getOne())
12961 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12962
12963 uint32_t Numerator = CaseProb.getNumerator();
12964 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
12965 return BranchProbability(Numerator, std::max(Numerator, Denominator));
12966}
12967
12968// Try to peel the top probability case if it exceeds the threshold.
12969// Return current MachineBasicBlock for the switch statement if the peeling
12970// does not occur.
12971// If the peeling is performed, return the newly created MachineBasicBlock
12972// for the peeled switch statement. Also update Clusters to remove the peeled
12973// case. PeeledCaseProb is the BranchProbability for the peeled case.
12974MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12975 const SwitchInst &SI, CaseClusterVector &Clusters,
12976 BranchProbability &PeeledCaseProb) {
12977 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12978 // Don't perform if there is only one cluster or optimizing for size.
12979 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12980 TM.getOptLevel() == CodeGenOptLevel::None ||
12981 SwitchMBB->getParent()->getFunction().hasMinSize())
12982 return SwitchMBB;
12983
12984 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
12985 unsigned PeeledCaseIndex = 0;
12986 bool SwitchPeeled = false;
12987 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12988 CaseCluster &CC = Clusters[Index];
12989 if (CC.Prob < TopCaseProb)
12990 continue;
12991 TopCaseProb = CC.Prob;
12992 PeeledCaseIndex = Index;
12993 SwitchPeeled = true;
12994 }
12995 if (!SwitchPeeled)
12996 return SwitchMBB;
12997
12998 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12999 << TopCaseProb << "\n");
13000
13001 // Record the MBB for the peeled switch statement.
13002 MachineFunction::iterator BBI(SwitchMBB);
13003 ++BBI;
13004 MachineBasicBlock *PeeledSwitchMBB =
13005 FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
13006 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
13007
13008 ExportFromCurrentBlock(SI.getCondition());
13009 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
13010 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
13011 nullptr, nullptr, TopCaseProb.getCompl()};
13012 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
13013
13014 Clusters.erase(PeeledCaseIt);
13015 for (CaseCluster &CC : Clusters) {
13016 LLVM_DEBUG(
13017 dbgs() << "Scale the probablity for one cluster, before scaling: "
13018 << CC.Prob << "\n");
13019 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
13020 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
13021 }
13022 PeeledCaseProb = TopCaseProb;
13023 return PeeledSwitchMBB;
13024}
13025
13026void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
13027 // Extract cases from the switch.
13028 BranchProbabilityInfo *BPI = FuncInfo.BPI;
13029 CaseClusterVector Clusters;
13030 Clusters.reserve(SI.getNumCases());
13031 for (auto I : SI.cases()) {
13032 MachineBasicBlock *Succ = FuncInfo.getMBB(I.getCaseSuccessor());
13033 const ConstantInt *CaseVal = I.getCaseValue();
13034 BranchProbability Prob =
13035 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
13036 : BranchProbability(1, SI.getNumCases() + 1);
13037 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
13038 }
13039
13040 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(SI.getDefaultDest());
13041
13042 // Cluster adjacent cases with the same destination. We do this at all
13043 // optimization levels because it's cheap to do and will make codegen faster
13044 // if there are many clusters.
13045 sortAndRangeify(Clusters);
13046
13047 // The branch probablity of the peeled case.
13048 BranchProbability PeeledCaseProb = BranchProbability::getZero();
13049 MachineBasicBlock *PeeledSwitchMBB =
13050 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
13051
13052 // If there is only the default destination, jump there directly.
13053 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
13054 if (Clusters.empty()) {
13055 assert(PeeledSwitchMBB == SwitchMBB);
13056 SwitchMBB->addSuccessor(DefaultMBB);
13057 if (DefaultMBB != NextBlock(SwitchMBB)) {
13058 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
13059 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
13060 }
13061 return;
13062 }
13063
13064 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
13065 DAG.getBFI());
13066 SL->findBitTestClusters(Clusters, &SI);
13067
13068 LLVM_DEBUG({
13069 dbgs() << "Case clusters: ";
13070 for (const CaseCluster &C : Clusters) {
13071 if (C.Kind == CC_JumpTable)
13072 dbgs() << "JT:";
13073 if (C.Kind == CC_BitTests)
13074 dbgs() << "BT:";
13075
13076 C.Low->getValue().print(dbgs(), true);
13077 if (C.Low != C.High) {
13078 dbgs() << '-';
13079 C.High->getValue().print(dbgs(), true);
13080 }
13081 dbgs() << ' ';
13082 }
13083 dbgs() << '\n';
13084 });
13085
13086 assert(!Clusters.empty());
13087 SwitchWorkList WorkList;
13088 CaseClusterIt First = Clusters.begin();
13089 CaseClusterIt Last = Clusters.end() - 1;
13090 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
13091 // Scale the branchprobability for DefaultMBB if the peel occurs and
13092 // DefaultMBB is not replaced.
13093 if (PeeledCaseProb != BranchProbability::getZero() &&
13094 DefaultMBB == FuncInfo.getMBB(SI.getDefaultDest()))
13095 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
13096 WorkList.push_back(
13097 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
13098
13099 while (!WorkList.empty()) {
13100 SwitchWorkListItem W = WorkList.pop_back_val();
13101 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
13102
13103 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
13104 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
13105 // For optimized builds, lower large range as a balanced binary tree.
13106 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
13107 continue;
13108 }
13109
13110 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
13111 }
13112}
13113
13114void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
13115 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13116 auto DL = getCurSDLoc();
13117 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13118 setValue(&I, DAG.getStepVector(DL, ResultVT));
13119}
13120
13121void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
13122 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13123 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13124
13125 SDLoc DL = getCurSDLoc();
13126 SDValue V = getValue(I.getOperand(0));
13127 assert(VT == V.getValueType() && "Malformed vector.reverse!");
13128
13129 if (VT.isScalableVector()) {
13130 setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
13131 return;
13132 }
13133
13134 // Use VECTOR_SHUFFLE for the fixed-length vector
13135 // to maintain existing behavior.
13136 SmallVector<int, 8> Mask;
13137 unsigned NumElts = VT.getVectorMinNumElements();
13138 for (unsigned i = 0; i != NumElts; ++i)
13139 Mask.push_back(NumElts - 1 - i);
13140
13141 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
13142}
13143
13144void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
13145 unsigned Factor) {
13146 auto DL = getCurSDLoc();
13147 SDValue InVec = getValue(I.getOperand(0));
13148
13149 SmallVector<EVT, 4> ValueVTs;
13150 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
13151 ValueVTs);
13152
13153 EVT OutVT = ValueVTs[0];
13154 unsigned OutNumElts = OutVT.getVectorMinNumElements();
13155
13156 SmallVector<SDValue, 4> SubVecs(Factor);
13157 for (unsigned i = 0; i != Factor; ++i) {
13158 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
13159 SubVecs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
13160 DAG.getVectorIdxConstant(OutNumElts * i, DL));
13161 }
13162
13163 SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
13164 DAG.getVTList(ValueVTs), SubVecs);
13165 setValue(&I, Res);
13166}
13167
13168void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
13169 unsigned Factor) {
13170 auto DL = getCurSDLoc();
13171 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13172 EVT InVT = getValue(I.getOperand(0)).getValueType();
13173 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13174
13175 SmallVector<SDValue, 8> InVecs(Factor);
13176 for (unsigned i = 0; i < Factor; ++i) {
13177 InVecs[i] = getValue(I.getOperand(i));
13178 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
13179 "Expected VTs to be the same");
13180 }
13181
13182 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
13183 SDValue Res =
13184 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);
13185
13187 for (unsigned i = 0; i < Factor; ++i)
13188 Results[i] = Res.getValue(i);
13189
13190 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Results);
13191 setValue(&I, Res);
13192}
13193
13194void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
13195 SmallVector<EVT, 4> ValueVTs;
13196 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
13197 ValueVTs);
13198 unsigned NumValues = ValueVTs.size();
13199 if (NumValues == 0) return;
13200
13202 SDValue Op = getValue(I.getOperand(0));
13203
13204 for (unsigned i = 0; i != NumValues; ++i)
13205 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
13206 SDValue(Op.getNode(), Op.getResNo() + i));
13207
13209 DAG.getVTList(ValueVTs), Values));
13210}
13211
13212void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
13213 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13214 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13215
13216 SDLoc DL = getCurSDLoc();
13217 SDValue V1 = getValue(I.getOperand(0));
13218 SDValue V2 = getValue(I.getOperand(1));
13219 const bool IsLeft = I.getIntrinsicID() == Intrinsic::vector_splice_left;
13220
13221 // VECTOR_SHUFFLE doesn't support a scalable or non-constant mask.
13222 if (VT.isScalableVector() || !isa<ConstantInt>(I.getOperand(2))) {
13223 SDValue Offset = DAG.getZExtOrTrunc(
13224 getValue(I.getOperand(2)), DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
13225 setValue(&I, DAG.getNode(IsLeft ? ISD::VECTOR_SPLICE_LEFT
13227 DL, VT, V1, V2, Offset));
13228 return;
13229 }
13230 uint64_t Imm = cast<ConstantInt>(I.getOperand(2))->getZExtValue();
13231
13232 unsigned NumElts = VT.getVectorNumElements();
13233
13234 uint64_t Idx = IsLeft ? Imm : NumElts - Imm;
13235
13236 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
13237 SmallVector<int, 8> Mask;
13238 for (unsigned i = 0; i < NumElts; ++i)
13239 Mask.push_back(Idx + i);
13240 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
13241}
13242
13243// Consider the following MIR after SelectionDAG, which produces output in
13244// phyregs in the first case or virtregs in the second case.
13245//
13246// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
13247// %5:gr32 = COPY $ebx
13248// %6:gr32 = COPY $edx
13249// %1:gr32 = COPY %6:gr32
13250// %0:gr32 = COPY %5:gr32
13251//
13252// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
13253// %1:gr32 = COPY %6:gr32
13254// %0:gr32 = COPY %5:gr32
13255//
13256// Given %0, we'd like to return $ebx in the first case and %5 in the second.
13257// Given %1, we'd like to return $edx in the first case and %6 in the second.
13258//
13259// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
13260// to a single virtreg (such as %0). The remaining outputs monotonically
13261// increase in virtreg number from there. If a callbr has no outputs, then it
13262// should not have a corresponding callbr landingpad; in fact, the callbr
13263// landingpad would not even be able to refer to such a callbr.
13265 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
13266 // There is definitely at least one copy.
13267 assert(MI->getOpcode() == TargetOpcode::COPY &&
13268 "start of copy chain MUST be COPY");
13269 Reg = MI->getOperand(1).getReg();
13270
13271 // If the copied register in the first copy must be virtual.
13272 assert(Reg.isVirtual() && "expected COPY of virtual register");
13273 MI = MRI.def_begin(Reg)->getParent();
13274
13275 // There may be an optional second copy.
13276 if (MI->getOpcode() == TargetOpcode::COPY) {
13277 assert(Reg.isVirtual() && "expected COPY of virtual register");
13278 Reg = MI->getOperand(1).getReg();
13279 assert(Reg.isPhysical() && "expected COPY of physical register");
13280 } else {
13281 // The start of the chain must be an INLINEASM_BR.
13282 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13283 "end of copy chain MUST be INLINEASM_BR");
13284 }
13285
13286 return Reg;
13287}
13288
13289// We must do this walk rather than the simpler
13290// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
13291// otherwise we will end up with copies of virtregs only valid along direct
13292// edges.
13293void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
13294 SmallVector<EVT, 8> ResultVTs;
13295 SmallVector<SDValue, 8> ResultValues;
13296 const auto *CBR =
13297 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
13298
13299 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13300 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
13301 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
13302
13303 Register InitialDef = FuncInfo.ValueMap[CBR];
13304 SDValue Chain = DAG.getRoot();
13305
13306 // Re-parse the asm constraints string.
13307 TargetLowering::AsmOperandInfoVector TargetConstraints =
13308 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
13309 for (auto &T : TargetConstraints) {
13310 SDISelAsmOperandInfo OpInfo(T);
13311 if (OpInfo.Type != InlineAsm::isOutput)
13312 continue;
13313
13314 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
13315 // individual constraint.
13316 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
13317
13318 switch (OpInfo.ConstraintType) {
13321 // Fill in OpInfo.AssignedRegs.Regs.
13322 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
13323
13324 // getRegistersForValue may produce 1 to many registers based on whether
13325 // the OpInfo.ConstraintVT is legal on the target or not.
13326 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
13327 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
13328 if (OriginalDef.isPhysical())
13329 FuncInfo.MBB->addLiveIn(OriginalDef);
13330 // Update the assigned registers to use the original defs.
13331 Reg = OriginalDef;
13332 }
13333
13334 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
13335 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
13336 ResultValues.push_back(V);
13337 ResultVTs.push_back(OpInfo.ConstraintVT);
13338 break;
13339 }
13341 SDValue Flag;
13343 OpInfo, DAG);
13344 ++InitialDef;
13345 ResultValues.push_back(V);
13346 ResultVTs.push_back(OpInfo.ConstraintVT);
13347 break;
13348 }
13349 default:
13350 break;
13351 }
13352 }
13354 DAG.getVTList(ResultVTs), ResultValues);
13355 setValue(&I, V);
13356}
unsigned NumParts
return SDValue()
static unsigned getIntrinsicID(const SDNode *N)
bool IsScalable
const TargetLowering & TLI
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
dxil translate DXIL Translate Metadata
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition FastISel.cpp:947
#define Check(C,...)
static Value * getCondition(Instruction *I)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
static void computeConstraintToUse(const TargetLowering *TLI, TargetLowering::AsmOperandInfo &OpInfo)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
static bool isUndef(const MachineInstr &MI)
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file contains the declarations for metadata subclasses.
Type::TypeID TypeID
#define T
#define T1
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
#define P(N)
if(PassOpts->AAPipeline)
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
MachineInstr *& Lo
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static void failForInvalidBundles(const CallBase &I, StringRef Name, ArrayRef< uint32_t > AllowedBundles)
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, SelectionDAG &DAG)
Prepare DAG-level operands.
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static const GlobalValue * getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx, const MachineFunction &MF)
If V is the address of a describable global, possibly displaced by a constant, return the global and ...
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
DenseMap< const Argument *, std::pair< const AllocaInst *, const StoreInst * > > ArgCopyElisionMapTy
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< Register, TypeSize > > &Regs, const SDValue &N)
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static bool determineConstraints(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, const TargetMachine &TM, SelectionDAG &DAG, const BasicBlock *EHPadBB)
DetermineConstraints - Find the constraints to use for inline asm operands.
static bool constructOperandInfo(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, SelectionDAGBuilder &Builder, const TargetLowering &TLI, ExtraFlags &ExtraInfo)
Construct operand info objects.
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts, MVT RegisterVT, MVT PartVT)
Emit a fatal error if the broken down registers don't match part type or count.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static FPClassTest getNoFPClass(const Instruction &I)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const uint8_t *MatcherTable, size_t &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
TTI::PartialReductionExtendKind ExtendKind
uint16_t RegSizeInBits(const MCRegisterInfo &MRI, MCRegister RegNo)
Value * RHS
Value * LHS
The Input class is used to parse a yaml document into in-memory structs and vectors.
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static LLVM_ABI Semantics SemanticsToEnum(const llvm::fltSemantics &Sem)
Definition APFloat.cpp:185
static LLVM_ABI const fltSemantics * getArbitraryFPSemantics(StringRef Format)
Returns the fltSemantics for a given arbitrary FP format string, or nullptr if invalid.
Definition APFloat.cpp:6340
Class for arbitrary precision integers.
Definition APInt.h:78
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:330
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition Function.cpp:332
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition Argument.h:50
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
iterator end() const
Definition ArrayRef.h:130
size_t size() const
Get the array size.
Definition ArrayRef.h:141
iterator begin() const
Definition ArrayRef.h:129
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:448
LLVM Basic Block Representation.
Definition BasicBlock.h:62
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
InstListType::const_iterator const_iterator
Definition BasicBlock.h:171
LLVM_ABI bool isEntryBlock() const
Return true if this is the entry block of the containing function.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
This class represents a no-op cast from one type to another.
The address of a basic block.
Definition Constants.h:1088
Analysis providing branch probability information.
LLVM_ABI BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
LLVM_ABI bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static constexpr BranchProbability getOne()
static uint32_t getDenominator()
static constexpr BranchProbability getUnknown()
static constexpr BranchProbability getZero()
uint32_t getNumerator() const
LLVM_ABI uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
LLVM_ABI bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition InstrTypes.h:728
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
Conditional Branch instruction.
Class for constant bytes.
Definition Constants.h:281
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition Constants.h:755
A constant value that is initialized with an expression using other constant values.
Definition Constants.h:1316
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
A signed pointer, in the ptrauth sense.
Definition Constants.h:1223
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition Constants.h:674
This is an important base class in LLVM.
Definition Constant.h:44
This is the common base class for constrained floating point intrinsics.
LLVM_ABI std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
LLVM_ABI unsigned getNonMetadataArgCount() const
DWARF expression.
LLVM_ABI bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static LLVM_ABI std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
LLVM_ABI uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static LLVM_ABI const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static LLVM_ABI DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
Base class for variables.
LLVM_ABI std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool isBigEndian() const
Definition DataLayout.h:221
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LLVM_ABI Value * getVariableLocationOp(unsigned OpIdx) const
DIExpression * getExpression() const
DILocalVariable * getVariable() const
LLVM_ABI iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
A debug info location.
Definition DebugLoc.h:126
LLVM_ABI DILocation * getInlinedAt() const
Definition DebugLoc.cpp:58
bool empty() const
Definition DenseMap.h:716
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT, true > const_iterator
Definition DenseMap.h:679
void reserve(size_type NumEntries)
Grow the DenseMap so that it can contain at least NumEntries items before resizing again.
Definition DenseMap.h:721
iterator end()
Definition DenseMap.h:686
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
Diagnostic information for inline asm reporting.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:312
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
Lightweight error class with error context and mandatory checking.
Definition Error.h:159
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition FastISel.h:67
bool allowReassoc() const
Flag queries.
Definition FMF.h:64
An instruction for ordering other memory operations.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
MachineBasicBlock * getMBB(const BasicBlock *BB) const
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
MachineBasicBlock * MBB
MBB - The current block.
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition Function.h:803
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:705
bool hasParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const
check if an attributes is in the list of attributes.
Definition Function.cpp:743
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:329
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition Function.h:252
size_t arg_size() const
Definition Function.h:895
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
Garbage collection metadata for a single function.
Definition GCMetadata.h:80
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
bool isInBounds() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
void setMemConstraint(ConstraintCode C)
setMemConstraint - Augment an existing flag with the constraint code for a memory constraint.
Definition InlineAsm.h:417
This instruction inserts a struct field of array element value into an aggregate value.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
A helper class to return the specified delimiter string after the first invocation of operator String...
An instruction for reading from memory.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
static LocationSize upperBound(uint64_t Value)
bool usesPerInvokeEHLabels(ExceptionHandling EH) const
Returns true if the exception tables reference the per-invoke EH labels emitted around invokes.
Definition MCAsmInfo.h:692
LLVM_ABI MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
LLVM_ABI MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
const MCAsmInfo & getAsmInfo() const
Definition MCContext.h:408
unsigned getID() const
getID() - Return the register class ID number.
bool isAllocatable() const
isAllocatable - Return true if this register class may be used to create virtual registers.
const MCPhysReg * iterator
iterator begin() const
begin/end - Return all of the registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
iterator end() const
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition MCSymbol.h:42
Metadata node.
Definition Metadata.h:1087
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1443
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:613
Machine Value Type.
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
std::string getString() const
Return the value type as a string, e.g. i32.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHContTarget(bool V=true)
Indicates if this is a target of Windows EH Continuation Guard.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
MachineInstrBundleIterator< MachineInstr > iterator
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
void setHasEHContTarget(bool V)
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI MCRegister getLiveInPhysReg(Register VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition MapVector.h:119
bool contains(const KeyT &Key) const
Definition MapVector.h:149
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:185
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Definition Module.cpp:377
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:882
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(Register VReg)
static SDDbgOperand fromConst(const Value *Const)
static SDDbgOperand fromGlobalAddr(const GlobalValue *GV)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const CondBrInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
DenseMap< const Constant *, Register > ConstantsOut
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
SDValue lowerStartEH(SDValue Chain, const BasicBlock *EHPadBB, MCSymbol *&BeginLabel)
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
const TargetTransformInfo * TTI
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
SDValue lowerNoFPClassToAssertNoFPClass(SelectionDAG &DAG, const Instruction &I, SDValue Op)
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void setValueToPoison(const Value *V, const SDLoc &dl)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, Register Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool canTailCall(const CallBase &CB) const
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void CopyValueToVirtualRegister(const Value *V, Register Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, const TargetLibraryInfo *li, const TargetTransformInfo &TTI)
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
SDValue getFPOperationRoot(fp::ExceptionBehavior EB)
Return the current virtual root of the Selection DAG, flushing PendingConstrainedFP or PendingConstra...
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineRegisterInfo * RegInfo
std::unique_ptr< SwiftErrorValueTracking > SwiftError
virtual void emitFunctionEntryCode()
std::unique_ptr< SelectionDAGBuilder > SDB
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemccpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue C, SDValue Size, const CallInst *CI) const
Emit target-specific code that performs a memccpy, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, const CallInst *CI) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrstr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, const CallInst *CI) const
Emit target-specific code that performs a strstr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo, const CallInst *CI) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, const CallInst *CI) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy, const CallInst *CI) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI bool shouldOptForSize() const
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
LLVM_ABI void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
LLVM_ABI SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void resize(size_type N)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitFunctionBasedCheckStackProtector() const
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
Multiway switch.
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Provides information about what library functions are available for the current target.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Returns the desired alignment for ByVal or InAlloca aggregate function arguments in the caller parame...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
virtual bool isAtomicAlignmentSupported(Align Alignment, uint64_t SizeInBytes) const
Return true if the target supports an atomic access of SizeInBytes bytes at the given Alignment.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
virtual Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual bool useStackGuardMixFP() const
If this function returns true, stack protection checks should mix the frame pointer (or whichever poi...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr, CodeGenOptLevel OptLevel=CodeGenOptLevel::Default) const
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
MachineMemOperand::Flags getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
virtual Value * getSDagStackGuard(const Module &M, const LibcallLoweringInfo &Libcalls) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
virtual Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_UDIV, ISD::VP_SDIV,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual SDValue emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
Definition Type.cpp:170
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:267
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
bool isTokenTy() const
Return true if this is 'token'.
Definition Type.h:231
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
Unconditional Branch instruction.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_iterator op_begin()
Definition User.h:259
unsigned getNumOperands() const
Definition User.h:229
op_iterator op_end()
Definition User.h:261
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
This is the common base class for vector predication intrinsics.
static LLVM_ABI std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
LLVM_ABI MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:707
bool use_empty() const
Definition Value.h:348
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
const ParentTy * getParent() const
Definition ilist_node.h:34
A raw_ostream that writes to an std::string.
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ CONVERGENCECTRL_ANCHOR
The llvm.experimental.convergence.* intrinsics.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition ISDOpcodes.h:47
@ SET_FPENV
Sets the current floating-point environment.
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ COND_LOOP
COND_LOOP is a conditional branch to self, used for implementing efficient conditional traps.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition ISDOpcodes.h:170
@ VECREDUCE_FMINIMUMNUM
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ STACKADDRESS
STACKADDRESS - Represents the llvm.stackaddress intrinsic.
Definition ISDOpcodes.h:129
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:397
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:403
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition ISDOpcodes.h:174
@ GlobalAddress
Definition ISDOpcodes.h:90
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
Definition ISDOpcodes.h:665
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
Definition ISDOpcodes.h:533
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
Definition ISDOpcodes.h:520
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:410
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition ISDOpcodes.h:158
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ SET_ROUNDING
Set rounding mode.
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition ISDOpcodes.h:119
@ CONVERGENCECTRL_ENTRY
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PARTIAL_REDUCE_FMLA
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:640
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
Definition ISDOpcodes.h:80
@ PtrAuthGlobalAddress
A ptrauth constant.
Definition ISDOpcodes.h:102
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition ISDOpcodes.h:50
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition ISDOpcodes.h:141
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition ISDOpcodes.h:137
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition ISDOpcodes.h:389
@ PATCHPOINT
The llvm.experimental.patchpoint.
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
Definition ISDOpcodes.h:669
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition ISDOpcodes.h:650
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:416
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition ISDOpcodes.h:152
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:112
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
Definition ISDOpcodes.h:807
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:505
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ RELOC_NONE
Issue a no-op relocation against a given symbol at the current location.
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition ISDOpcodes.h:759
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
Definition ISDOpcodes.h:673
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition ISDOpcodes.h:430
@ STACKMAP
The llvm.experimental.stackmap intrinsic.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition ISDOpcodes.h:715
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition ISDOpcodes.h:124
@ CLEAR_CACHE
llvm.clear_cache intrinsic Operands: Input Chain, Start Addres, End Address Outputs: Output Chain
@ CONVERGENCECTRL_LOOP
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ VECREDUCE_FMINIMUM
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition ISDOpcodes.h:164
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ BRCOND
BRCOND - Conditional branch.
@ VECREDUCE_SEQ_FMUL
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
Definition ISDOpcodes.h:645
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:629
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ CTTZ_ELTS_ZERO_POISON
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
Definition ISDOpcodes.h:775
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Flag
These should be considered private to the implementation of the MCInstrDesc class.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_VScale()
Matches a call to llvm.vscale().
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
Offsets
Offsets in bytes from the start of the input buffer.
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
LLVM_ABI void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
std::vector< CaseCluster > CaseClusterVector
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
CaseClusterVector::iterator CaseClusterIt
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition Dwarf.h:149
ExceptionBehavior
Exception behavior used for floating point operations.
Definition FPEnv.h:39
@ ebStrict
This corresponds to "fpexcept.strict".
Definition FPEnv.h:42
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition FPEnv.h:41
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition FPEnv.h:40
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)
Extract a Value from Metadata, allowing null.
Definition Metadata.h:695
constexpr float log2ef
Definition MathExtras.h:52
constexpr double e
constexpr float ln2f
Definition MathExtras.h:50
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:339
LLVM_ABI ISD::NodeType getAtomicRMWOpcode(AtomicRMWInst::BinOp Op)
Return the atomic node opcode for an atomicrmw operation.
Definition Analysis.cpp:292
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
LLVM_ABI bool canDescribeGlobalAddressInLocationList(const MachineFunction &MF)
Test if the debug info for MF can name a describable global address in a location list too,...
Definition Analysis.cpp:695
LLVM_ABI ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition Analysis.cpp:248
LLVM_ABI cl::opt< bool > ProfcheckDisableMetadataFixes
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
SDValue peekThroughFreeze(SDValue V)
Return the non-frozen source operand of V if it exists.
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
RelativeUniformCounterPtr Values
Definition InstrProf.h:91
@ Unknown
Not known to have no common set bits.
LLVM_ABI void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs=nullptr, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition Analysis.cpp:121
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Done
Definition Threading.h:60
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
LLVM_ABI void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
LLVM_ABI bool isExceptionPointerAndSelectorType(Type *Ty)
Return true if landingpad result type Ty is a struct of an exception pointer (pointer or integer) and...
Definition Analysis.cpp:201
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
static ConstantRange getRange(Value *Op, SCCPSolver &Solver, const SmallPtrSetImpl< Value * > &InsertedValues)
Helper for getting ranges from Solver.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
auto cast_or_null(const Y &Val)
Definition Casting.h:714
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
gep_type_iterator gep_type_end(const User *GEP)
LLVM_ABI LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
Definition STLExtras.h:2189
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
Definition STLExtras.h:1167
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
LLVM_ABI void ComputeValueTypes(const DataLayout &DL, Type *Ty, SmallVectorImpl< Type * > &Types, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
Given an LLVM IR type, compute non-aggregate subtypes.
Definition Analysis.cpp:74
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition STLExtras.h:869
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
generic_gep_type_iterator<> gep_type_iterator
auto succ_size(const MachineBasicBlock *BB)
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition STLExtras.h:300
LLVM_ABI ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition Analysis.cpp:214
LLVM_ABI MDNode * getMemCacheHint(const Instruction &I, unsigned OperandNo=0)
Return the hint payload for operand OperandNo of I, or nullptr if absent.
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition Local.cpp:2353
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
LLVM_ABI bool isOnlyUsedInZeroEqualityComparison(const Instruction *CtxI)
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
LLVM_ABI bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition Analysis.cpp:709
DWARFExpression::Operation Op
@ Dynamic
Denotes mode unknown at compile time.
LLVM_ABI ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition Analysis.cpp:236
LLVM_ABI const GlobalValue * getDescribableGlobalAddress(const Constant *C, int64_t &Offset, const MachineFunction &MF)
If C is the address of a global, possibly displaced by a constant, return that global and set Offset ...
Definition Analysis.cpp:676
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
LLVM_ABI std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition FPEnv.cpp:25
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
LLVM_ABI GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition Analysis.cpp:183
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
LLVM_ABI Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
LLVM_ABI unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition Analysis.cpp:35
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:347
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
#define NC
Definition regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
uint64_t getScalarStoreSize() const
Definition ValueTypes.h:440
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:98
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
Definition ValueTypes.h:212
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isFixedLengthVector() const
Definition ValueTypes.h:214
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:330
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
Definition ValueTypes.h:136
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition InlineAsm.h:128
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:262
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
A lightweight accessor for an operand bundle meant to be passed around by value.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< std::pair< Register, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< Register, 4 > Regs
This list holds the registers assigned to the values.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
void setUnpredictable(bool b)
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition MapVector.h:344
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
Register Reg
The virtual register containing the index of the jump table entry to jump to.
MachineBasicBlock * Default
The MBB of the default bb, which is a successor of the range check MBB.
unsigned JTI
The JumpTableIndex for this jump table in the function.
MachineBasicBlock * MBB
The MBB into which to emit the code for the indirect jump.
std::optional< SDLoc > SL
The debug location of the instruction this JumpTable was produced from.
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setDeactivationSymbol(GlobalValue *Sym)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
Type * OrigRetTy
Original unlegalized return type.
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
LLVM_ABI void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)