LLVM 24.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsBPF.h"
82#include "llvm/IR/IntrinsicsWebAssembly.h"
83#include "llvm/IR/LLVMContext.h"
86#include "llvm/IR/Metadata.h"
87#include "llvm/IR/Module.h"
88#include "llvm/IR/Operator.h"
91#include "llvm/IR/Statepoint.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCAsmInfo.h"
96#include "llvm/MC/MCContext.h"
101#include "llvm/Support/Debug.h"
109#include <cstddef>
110#include <limits>
111#include <optional>
112#include <tuple>
113
114using namespace llvm;
115using namespace PatternMatch;
116using namespace SwitchCG;
117
118#define DEBUG_TYPE "isel"
119
120/// LimitFloatPrecision - Generate low-precision inline sequences for
121/// some float libcalls (6, 8 or 12 bits).
122static unsigned LimitFloatPrecision;
123
124static cl::opt<bool>
125 InsertAssertAlign("insert-assert-align", cl::init(true),
126 cl::desc("Insert the experimental `assertalign` node."),
128
130 LimitFPPrecision("limit-float-precision",
131 cl::desc("Generate low-precision inline sequences "
132 "for some float libcalls"),
134 cl::init(0));
135
137 "switch-peel-threshold", cl::Hidden, cl::init(66),
138 cl::desc("Set the case probability threshold for peeling the case from a "
139 "switch statement. A value greater than 100 will void this "
140 "optimization"));
141
142// Limit the width of DAG chains. This is important in general to prevent
143// DAG-based analysis from blowing up. For example, alias analysis and
144// load clustering may not complete in reasonable time. It is difficult to
145// recognize and avoid this situation within each individual analysis, and
146// future analyses are likely to have the same behavior. Limiting DAG width is
147// the safe approach and will be especially important with global DAGs.
148//
149// MaxParallelChains default is arbitrarily high to avoid affecting
150// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
151// sequence over this should have been converted to llvm.memcpy by the
152// frontend. It is easy to induce this behavior with .ll code such as:
153// %buffer = alloca [4096 x i8]
154// %data = load [4096 x i8]* %argPtr
155// store [4096 x i8] %data, [4096 x i8]* %buffer
156static const unsigned MaxParallelChains = 64;
157
159 const SDValue *Parts, unsigned NumParts,
160 MVT PartVT, EVT ValueVT, const Value *V,
161 SDValue InChain,
162 std::optional<CallingConv::ID> CC);
163
164/// getCopyFromParts - Create a value that contains the specified legal parts
165/// combined into the value they represent. If the parts combine to a type
166/// larger than ValueVT then AssertOp can be used to specify whether the extra
167/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
168/// (ISD::AssertSext).
169static SDValue
170getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
171 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
172 SDValue InChain,
173 std::optional<CallingConv::ID> CC = std::nullopt,
174 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
175 // Let the target assemble the parts if it wants to
177 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
178 PartVT, ValueVT, CC))
179 return Val;
180
181 if (ValueVT.isVector())
182 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
183 InChain, CC);
184
185 assert(NumParts > 0 && "No parts to assemble!");
186 SDValue Val = Parts[0];
187
188 if (NumParts > 1) {
189 // Assemble the value from multiple parts.
190 if (ValueVT.isInteger()) {
191 unsigned PartBits = PartVT.getSizeInBits();
192 unsigned ValueBits = ValueVT.getSizeInBits();
193
194 // Assemble the power of 2 part.
195 unsigned RoundParts = llvm::bit_floor(NumParts);
196 unsigned RoundBits = PartBits * RoundParts;
197 EVT RoundVT = RoundBits == ValueBits ?
198 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
199 SDValue Lo, Hi;
200
201 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
202
203 if (RoundParts > 2) {
204 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
205 InChain);
206 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
207 PartVT, HalfVT, V, InChain);
208 } else {
209 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
210 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
211 }
212
213 if (DAG.getDataLayout().isBigEndian())
214 std::swap(Lo, Hi);
215
216 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
217
218 if (RoundParts < NumParts) {
219 // Assemble the trailing non-power-of-2 part.
220 unsigned OddParts = NumParts - RoundParts;
221 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
222 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
223 OddVT, V, InChain, CC);
224
225 // Combine the round and odd parts.
226 Lo = Val;
227 if (DAG.getDataLayout().isBigEndian())
228 std::swap(Lo, Hi);
229 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
230 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
231 Hi = DAG.getNode(
232 ISD::SHL, DL, TotalVT, Hi,
233 DAG.getShiftAmountConstant(Lo.getValueSizeInBits(), TotalVT, DL));
234 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
235 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
236 }
237 } else if (PartVT.isFloatingPoint()) {
238 // FP split into multiple FP parts (for ppcf128)
239 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
240 "Unexpected split");
241 SDValue Lo, Hi;
242 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
243 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
244 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
245 std::swap(Lo, Hi);
246 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
247 } else {
248 // FP split into integer parts (soft fp)
249 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
250 !PartVT.isVector() && "Unexpected split");
251 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
252 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
253 InChain, CC);
254 }
255 }
256
257 // There is now one part, held in Val. Correct it to match ValueVT.
258 // PartEVT is the type of the register class that holds the value.
259 // ValueVT is the type of the inline asm operation.
260 EVT PartEVT = Val.getValueType();
261
262 if (PartEVT == ValueVT)
263 return Val;
264
265 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
266 ValueVT.bitsLT(PartEVT)) {
267 // For an FP value in an integer part, we need to truncate to the right
268 // width first.
269 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
270 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
271 }
272
273 // Handle types that have the same size.
274 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
275 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
276
277 // Handle types with different sizes.
278 if (PartEVT.isInteger() && ValueVT.isInteger()) {
279 if (ValueVT.bitsLT(PartEVT)) {
280 // For a truncate, see if we have any information to
281 // indicate whether the truncated bits will always be
282 // zero or sign-extension.
283 if (AssertOp)
284 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
285 DAG.getValueType(ValueVT));
286 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
287 }
288 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
289 }
290
291 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
292 // FP_ROUND's are always exact here.
293 if (ValueVT.bitsLT(Val.getValueType())) {
294
295 SDValue NoChange =
296 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
297
298 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
299 llvm::Attribute::StrictFP)) {
300 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
301 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
302 NoChange);
303 }
304
305 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
306 }
307
308 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
309 }
310
311 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
312 // then truncating.
313 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
314 ValueVT.bitsLT(PartEVT)) {
315 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
316 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
317 }
318
319 report_fatal_error("Unknown mismatch in getCopyFromParts!");
320}
321
323 const Twine &ErrMsg) {
325 if (!I)
326 return Ctx.emitError(ErrMsg);
327
328 if (const CallInst *CI = dyn_cast<CallInst>(I))
329 if (CI->isInlineAsm()) {
330 return Ctx.diagnose(DiagnosticInfoInlineAsm(
331 *CI, ErrMsg + ", possible invalid constraint for vector type"));
332 }
333
334 return Ctx.emitError(I, ErrMsg);
335}
336
337/// Emit a fatal error if the broken down registers don't match part type
338/// or count.
339static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts,
340 MVT RegisterVT, MVT PartVT) {
341 if (NumRegs == NumParts && RegisterVT == PartVT)
342 return;
343
345 Twine("Part count doesn't match vector breakdown! ") + Twine(NumRegs) +
346 " registers, " + Twine(NumParts) + " parts, " + RegisterVT.getString() +
347 " RegisterVT, " + PartVT.getString() + " PartVT");
348}
349
350/// getCopyFromPartsVector - Create a value that contains the specified legal
351/// parts combined into the value they represent. If the parts combine to a
352/// type larger than ValueVT then AssertOp can be used to specify whether the
353/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
354/// ValueVT (ISD::AssertSext).
356 const SDValue *Parts, unsigned NumParts,
357 MVT PartVT, EVT ValueVT, const Value *V,
358 SDValue InChain,
359 std::optional<CallingConv::ID> CallConv) {
360 assert(ValueVT.isVector() && "Not a vector value");
361 assert(NumParts > 0 && "No parts to assemble!");
362 const bool IsABIRegCopy = CallConv.has_value();
363
365 SDValue Val = Parts[0];
366
367 // Handle a multi-element vector.
368 if (NumParts > 1) {
369 EVT IntermediateVT;
370 MVT RegisterVT;
371 unsigned NumIntermediates;
372 unsigned NumRegs;
373
374 if (IsABIRegCopy) {
375 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
376 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
377 NumIntermediates, RegisterVT);
378 } else {
379 NumRegs =
380 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
381 NumIntermediates, RegisterVT);
382 }
383
384 ensureMatchedVecRegParts(NumRegs, NumParts, RegisterVT, PartVT);
385 assert(RegisterVT.getSizeInBits() ==
386 Parts[0].getSimpleValueType().getSizeInBits() &&
387 "Part type sizes don't match!");
388
389 // Assemble the parts into intermediate operands.
390 SmallVector<SDValue, 8> Ops(NumIntermediates);
391 if (NumIntermediates == NumParts) {
392 // If the register was not expanded, truncate or copy the value,
393 // as appropriate.
394 for (unsigned i = 0; i != NumParts; ++i)
395 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
396 V, InChain, CallConv);
397 } else if (NumParts > 0) {
398 // If the intermediate type was expanded, build the intermediate
399 // operands from the parts.
400 assert(NumParts % NumIntermediates == 0 &&
401 "Must expand into a divisible number of parts!");
402 unsigned Factor = NumParts / NumIntermediates;
403 for (unsigned i = 0; i != NumIntermediates; ++i)
404 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
405 IntermediateVT, V, InChain, CallConv);
406 }
407
408 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
409 // intermediate operands.
410 EVT BuiltVectorTy =
411 IntermediateVT.isVector()
413 *DAG.getContext(), IntermediateVT.getScalarType(),
414 IntermediateVT.getVectorElementCount() * NumParts)
416 IntermediateVT.getScalarType(),
417 NumIntermediates);
418 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
420 DL, BuiltVectorTy, Ops);
421 }
422
423 // There is now one part, held in Val. Correct it to match ValueVT.
424 EVT PartEVT = Val.getValueType();
425
426 if (PartEVT == ValueVT)
427 return Val;
428
429 if (PartEVT.isVector()) {
430 // Vector/Vector bitcast.
431 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
432 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
433
434 // If the parts vector has more elements than the value vector, then we
435 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
436 // Extract the elements we want.
437 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
440 (PartEVT.getVectorElementCount().isScalable() ==
441 ValueVT.getVectorElementCount().isScalable()) &&
442 "Cannot narrow, it would be a lossy transformation");
443 PartEVT =
445 ValueVT.getVectorElementCount());
446 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
447 DAG.getVectorIdxConstant(0, DL));
448 if (PartEVT == ValueVT)
449 return Val;
450 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
451 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
452
453 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
454 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
455 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
456 }
457
458 // Promoted vector extract
459 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
460 }
461
462 // Trivial bitcast if the types are the same size and the destination
463 // vector type is legal.
464 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
465 TLI.isTypeLegal(ValueVT))
466 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
467
468 if (ValueVT.getVectorNumElements() != 1) {
469 // Certain ABIs require that vectors are passed as integers. For vectors
470 // are the same size, this is an obvious bitcast.
471 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
472 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
473 } else if (ValueVT.bitsLT(PartEVT)) {
474 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
475 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
476 // Drop the extra bits.
477 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
478 return DAG.getBitcast(ValueVT, Val);
479 }
480
482 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
483 return DAG.getUNDEF(ValueVT);
484 }
485
486 // Handle cases such as i8 -> <1 x i1>
487 EVT ValueSVT = ValueVT.getVectorElementType();
488 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
489 unsigned ValueSize = ValueSVT.getSizeInBits();
490 if (ValueSize == PartEVT.getSizeInBits()) {
491 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
492 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
493 // It's possible a scalar floating point type gets softened to integer and
494 // then promoted to a larger integer. If PartEVT is the larger integer
495 // we need to truncate it and then bitcast to the FP type.
496 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
497 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
498 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
499 Val = DAG.getBitcast(ValueSVT, Val);
500 } else {
501 Val = ValueVT.isFloatingPoint()
502 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
503 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
504 }
505 }
506
507 return DAG.getBuildVector(ValueVT, DL, Val);
508}
509
510static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
511 SDValue Val, SDValue *Parts, unsigned NumParts,
512 MVT PartVT, const Value *V,
513 std::optional<CallingConv::ID> CallConv);
514
515/// getCopyToParts - Create a series of nodes that contain the specified value
516/// split into legal parts. If the parts contain more bits than Val, then, for
517/// integers, ExtendKind can be used to specify how to generate the extra bits.
518static void
520 unsigned NumParts, MVT PartVT, const Value *V,
521 std::optional<CallingConv::ID> CallConv = std::nullopt,
523 // Let the target split the parts if it wants to
525 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
526 CallConv))
527 return;
528 EVT ValueVT = Val.getValueType();
529
530 // Handle the vector case separately.
531 if (ValueVT.isVector())
532 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
533 CallConv);
534
535 unsigned OrigNumParts = NumParts;
537 "Copying to an illegal type!");
538
539 if (NumParts == 0)
540 return;
541
542 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
543 EVT PartEVT = PartVT;
544 if (PartEVT == ValueVT) {
545 assert(NumParts == 1 && "No-op copy with multiple parts!");
546 Parts[0] = Val;
547 return;
548 }
549
550 unsigned PartBits = PartVT.getSizeInBits();
551 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
552 // If the parts cover more bits than the value has, promote the value.
553 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
554 assert(NumParts == 1 && "Do not know what to promote to!");
555 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
556 } else {
557 if (ValueVT.isFloatingPoint()) {
558 // FP values need to be bitcast, then extended if they are being put
559 // into a larger container.
560 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
561 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
562 }
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
567 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
570 }
571 } else if (PartBits == ValueVT.getSizeInBits()) {
572 // Different types of the same size.
573 assert(NumParts == 1 && PartEVT != ValueVT);
574 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
575 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
576 // If the parts cover less bits than value has, truncate the value.
577 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
578 ValueVT.isInteger() &&
579 "Unknown mismatch!");
580 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
581 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
582 if (PartVT == MVT::x86mmx)
583 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
584 }
585
586 // The value may have changed - recompute ValueVT.
587 ValueVT = Val.getValueType();
588 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
589 "Failed to tile the value with PartVT!");
590
591 if (NumParts == 1) {
592 if (PartEVT != ValueVT) {
594 "scalar-to-vector conversion failed");
595 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
596 }
597
598 Parts[0] = Val;
599 return;
600 }
601
602 // Expand the value into multiple parts.
603 if (NumParts & (NumParts - 1)) {
604 // The number of parts is not a power of 2. Split off and copy the tail.
605 assert(PartVT.isInteger() && ValueVT.isInteger() &&
606 "Do not know what to expand to!");
607 unsigned RoundParts = llvm::bit_floor(NumParts);
608 unsigned RoundBits = RoundParts * PartBits;
609 unsigned OddParts = NumParts - RoundParts;
610 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
611 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
612
613 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
614 CallConv);
615
616 if (DAG.getDataLayout().isBigEndian())
617 // The odd parts were reversed by getCopyToParts - unreverse them.
618 std::reverse(Parts + RoundParts, Parts + NumParts);
619
620 NumParts = RoundParts;
621 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
622 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
623 }
624
625 // The number of parts is a power of 2. Repeatedly bisect the value using
626 // EXTRACT_ELEMENT.
627 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
629 ValueVT.getSizeInBits()),
630 Val);
631
632 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
633 for (unsigned i = 0; i < NumParts; i += StepSize) {
634 unsigned ThisBits = StepSize * PartBits / 2;
635 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
636 SDValue &Part0 = Parts[i];
637 SDValue &Part1 = Parts[i+StepSize/2];
638
639 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
640 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
641 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
642 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
643
644 if (ThisBits == PartBits && ThisVT != PartVT) {
645 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
646 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
647 }
648 }
649 }
650
651 if (DAG.getDataLayout().isBigEndian())
652 std::reverse(Parts, Parts + OrigNumParts);
653}
654
656 const SDLoc &DL, EVT PartVT) {
657 if (!PartVT.isVector())
658 return SDValue();
659
660 EVT ValueVT = Val.getValueType();
661 EVT PartEVT = PartVT.getVectorElementType();
662 EVT ValueEVT = ValueVT.getVectorElementType();
663 ElementCount PartNumElts = PartVT.getVectorElementCount();
664 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
665
666 // We only support widening vectors with equivalent element types and
667 // fixed/scalable properties. If a target needs to widen a fixed-length type
668 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
669 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
670 PartNumElts.isScalable() != ValueNumElts.isScalable())
671 return SDValue();
672
673 // Have a try for bf16 because some targets share its ABI with fp16.
674 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
676 "Cannot widen to illegal type");
677 Val = DAG.getNode(
679 ValueVT.changeVectorElementType(*DAG.getContext(), MVT::f16), Val);
680 } else if (PartEVT != ValueEVT) {
681 return SDValue();
682 }
683
684 // Widening a scalable vector to another scalable vector is done by inserting
685 // the vector into a larger undef one.
686 if (PartNumElts.isScalable())
687 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
688 Val, DAG.getVectorIdxConstant(0, DL));
689
690 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
691 // undef elements.
693 DAG.ExtractVectorElements(Val, Ops);
694 SDValue EltUndef = DAG.getUNDEF(PartEVT);
695 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
696
697 // FIXME: Use CONCAT for 2x -> 4x.
698 return DAG.getBuildVector(PartVT, DL, Ops);
699}
700
701/// getCopyToPartsVector - Create a series of nodes that contain the specified
702/// value split into legal parts.
703static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
704 SDValue Val, SDValue *Parts, unsigned NumParts,
705 MVT PartVT, const Value *V,
706 std::optional<CallingConv::ID> CallConv) {
707 EVT ValueVT = Val.getValueType();
708 assert(ValueVT.isVector() && "Not a vector");
710 const bool IsABIRegCopy = CallConv.has_value();
711
712 if (NumParts == 1) {
713 EVT PartEVT = PartVT;
714 if (PartEVT == ValueVT) {
715 // Nothing to do.
716 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
717 // Bitconvert vector->vector case.
718 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
719 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
720 Val = Widened;
721 } else if (PartVT.isVector() &&
723 ValueVT.getVectorElementType()) &&
724 PartEVT.getVectorElementCount() ==
725 ValueVT.getVectorElementCount()) {
726
727 // Promoted vector extract
728 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
729 } else if (PartEVT.isVector() &&
730 PartEVT.getVectorElementType() !=
731 ValueVT.getVectorElementType() &&
732 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
734 // Combination of widening and promotion.
735 EVT WidenVT =
737 PartVT.getVectorElementCount());
738 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
739 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
740 } else {
741 // Don't extract an integer from a float vector. This can happen if the
742 // FP type gets softened to integer and then promoted. The promotion
743 // prevents it from being picked up by the earlier bitcast case.
744 if (ValueVT.getVectorElementCount().isScalar() &&
745 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
746 // If we reach this condition and PartVT is FP, this means that
747 // ValueVT is also FP and both have a different size, otherwise we
748 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
749 // would be invalid since that would mean the smaller FP type has to
750 // be extended to the larger one.
751 if (PartVT.isFloatingPoint()) {
752 Val = DAG.getBitcast(ValueVT.getScalarType(), Val);
753 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
754 } else
755 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
756 DAG.getVectorIdxConstant(0, DL));
757 } else {
758 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
759 assert(PartVT.getFixedSizeInBits() > ValueSize &&
760 "lossy conversion of vector to scalar type");
761 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
762 Val = DAG.getBitcast(IntermediateType, Val);
763 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
764 }
765 }
766
767 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
768 Parts[0] = Val;
769 return;
770 }
771
772 // Handle a multi-element vector.
773 EVT IntermediateVT;
774 MVT RegisterVT;
775 unsigned NumIntermediates;
776 unsigned NumRegs;
777 if (IsABIRegCopy) {
778 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
779 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
780 RegisterVT);
781 } else {
782 NumRegs =
783 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
784 NumIntermediates, RegisterVT);
785 }
786
787 ensureMatchedVecRegParts(NumRegs, NumParts, RegisterVT, PartVT);
788 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
789 "Mixing scalable and fixed vectors when copying in parts");
790
791 std::optional<ElementCount> DestEltCnt;
792
793 if (IntermediateVT.isVector())
794 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
795 else
796 DestEltCnt = ElementCount::getFixed(NumIntermediates);
797
798 EVT BuiltVectorTy = EVT::getVectorVT(
799 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
800
801 if (ValueVT == BuiltVectorTy) {
802 // Nothing to do.
803 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
804 // Bitconvert vector->vector case.
805 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
806 } else {
807 if (BuiltVectorTy.getVectorElementType().bitsGT(
808 ValueVT.getVectorElementType())) {
809 // Integer promotion.
810 ValueVT = EVT::getVectorVT(*DAG.getContext(),
811 BuiltVectorTy.getVectorElementType(),
812 ValueVT.getVectorElementCount());
813 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
814 }
815
816 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
817 Val = Widened;
818 }
819 }
820
821 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
822
823 // Split the vector into intermediate operands.
824 SmallVector<SDValue, 8> Ops(NumIntermediates);
825 for (unsigned i = 0; i != NumIntermediates; ++i) {
826 if (IntermediateVT.isVector()) {
827 // This does something sensible for scalable vectors - see the
828 // definition of EXTRACT_SUBVECTOR for further details.
829 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
830 Ops[i] =
831 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
832 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
833 } else {
834 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
835 DAG.getVectorIdxConstant(i, DL));
836 }
837 }
838
839 // Split the intermediate operands into legal parts.
840 if (NumParts == NumIntermediates) {
841 // If the register was not expanded, promote or copy the value,
842 // as appropriate.
843 for (unsigned i = 0; i != NumParts; ++i)
844 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
845 } else if (NumParts > 0) {
846 // If the intermediate type was expanded, split each the value into
847 // legal parts.
848 assert(NumIntermediates != 0 && "division by zero");
849 assert(NumParts % NumIntermediates == 0 &&
850 "Must expand into a divisible number of parts!");
851 unsigned Factor = NumParts / NumIntermediates;
852 for (unsigned i = 0; i != NumIntermediates; ++i)
853 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
854 CallConv);
855 }
856}
857
858static void failForInvalidBundles(const CallBase &I, StringRef Name,
859 ArrayRef<uint32_t> AllowedBundles) {
860 if (I.hasOperandBundlesOtherThan(AllowedBundles)) {
861 ListSeparator LS;
862 std::string Error;
864 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
865 OperandBundleUse U = I.getOperandBundleAt(i);
866 if (!is_contained(AllowedBundles, U.getTagID()))
867 OS << LS << U.getTagName();
868 }
870 Twine("cannot lower ", Name)
871 .concat(Twine(" with arbitrary operand bundles: ", Error)));
872 }
873}
874
876 EVT valuevt, std::optional<CallingConv::ID> CC)
877 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
878 RegCount(1, regs.size()), CallConv(CC) {}
879
881 const DataLayout &DL, Register Reg, Type *Ty,
882 std::optional<CallingConv::ID> CC) {
884
885 CallConv = CC;
886
887 for (EVT ValueVT : ValueVTs) {
888 unsigned NumRegs =
890 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
891 : TLI.getNumRegisters(Context, ValueVT);
892 MVT RegisterVT =
894 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
895 : TLI.getRegisterType(Context, ValueVT);
896 for (unsigned i = 0; i != NumRegs; ++i)
897 Regs.push_back(Reg + i);
898 RegVTs.push_back(RegisterVT);
899 RegCount.push_back(NumRegs);
900 Reg = Reg.id() + NumRegs;
901 }
902}
903
905 FunctionLoweringInfo &FuncInfo,
906 const SDLoc &dl, SDValue &Chain,
907 SDValue *Glue, const Value *V) const {
908 // A Value with type {} or [0 x %t] needs no registers.
909 if (ValueVTs.empty())
910 return SDValue();
911
913
914 // Assemble the legal parts into the final values.
917 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
918 // Copy the legal parts from the registers.
919 EVT ValueVT = ValueVTs[Value];
920 unsigned NumRegs = RegCount[Value];
921 MVT RegisterVT = isABIMangled()
922 ? TLI.getRegisterTypeForCallingConv(
923 *DAG.getContext(), *CallConv, RegVTs[Value])
924 : RegVTs[Value];
925
926 Parts.resize(NumRegs);
927 for (unsigned i = 0; i != NumRegs; ++i) {
928 SDValue P;
929 if (!Glue) {
930 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
931 } else {
932 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
933 *Glue = P.getValue(2);
934 }
935
936 Chain = P.getValue(1);
937 Parts[i] = P;
938
939 // If the source register was virtual and if we know something about it,
940 // add an assert node.
941 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
942 continue;
943
945 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
946 if (!LOI)
947 continue;
948
949 unsigned RegSize = RegisterVT.getScalarSizeInBits();
950 unsigned NumSignBits = LOI->NumSignBits;
951 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
952
953 if (NumZeroBits == RegSize) {
954 // The current value is a zero.
955 // Explicitly express that as it would be easier for
956 // optimizations to kick in.
957 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
958 continue;
959 }
960
961 // FIXME: We capture more information than the dag can represent. For
962 // now, just use the tightest assertzext/assertsext possible.
963 bool isSExt;
964 EVT FromVT(MVT::Other);
965 if (NumZeroBits) {
966 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
967 isSExt = false;
968 } else if (NumSignBits > 1) {
969 FromVT =
970 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
971 isSExt = true;
972 } else {
973 continue;
974 }
975 // Add an assertion node.
976 assert(FromVT != MVT::Other);
977 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
978 RegisterVT, P, DAG.getValueType(FromVT));
979 }
980
981 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
982 RegisterVT, ValueVT, V, Chain, CallConv);
983 Part += NumRegs;
984 Parts.clear();
985 }
986
987 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
988}
989
991 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
992 const Value *V,
993 ISD::NodeType PreferredExtendType) const {
995 ISD::NodeType ExtendKind = PreferredExtendType;
996
997 // Get the list of the values's legal parts.
998 unsigned NumRegs = Regs.size();
999 SmallVector<SDValue, 8> Parts(NumRegs);
1000 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
1001 unsigned NumParts = RegCount[Value];
1002
1003 MVT RegisterVT = isABIMangled()
1004 ? TLI.getRegisterTypeForCallingConv(
1005 *DAG.getContext(), *CallConv, RegVTs[Value])
1006 : RegVTs[Value];
1007
1009 if (TLI.isZExtFree(peekThroughFreeze(Val), RegisterVT))
1011
1012 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
1013 NumParts, RegisterVT, V, CallConv, ExtendKind);
1014 Part += NumParts;
1015 }
1016
1017 // Copy the parts into the registers.
1018 SmallVector<SDValue, 8> Chains(NumRegs);
1019 for (unsigned i = 0; i != NumRegs; ++i) {
1020 SDValue Part;
1021 if (!Glue) {
1022 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
1023 } else {
1024 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
1025 *Glue = Part.getValue(1);
1026 }
1027
1028 Chains[i] = Part.getValue(0);
1029 }
1030
1031 if (NumRegs == 1 || Glue)
1032 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1033 // flagged to it. That is the CopyToReg nodes and the user are considered
1034 // a single scheduling unit. If we create a TokenFactor and return it as
1035 // chain, then the TokenFactor is both a predecessor (operand) of the
1036 // user as well as a successor (the TF operands are flagged to the user).
1037 // c1, f1 = CopyToReg
1038 // c2, f2 = CopyToReg
1039 // c3 = TokenFactor c1, c2
1040 // ...
1041 // = op c3, ..., f2
1042 Chain = Chains[NumRegs-1];
1043 else
1044 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1045}
1046
1048 unsigned MatchingIdx, const SDLoc &dl,
1049 SelectionDAG &DAG,
1050 std::vector<SDValue> &Ops) const {
1052
1053 InlineAsm::Flag Flag(Code, Regs.size());
1054 if (HasMatching)
1055 Flag.setMatchingOp(MatchingIdx);
1056 else if (!Regs.empty() && Regs.front().isVirtual()) {
1057 // Put the register class of the virtual registers in the flag word. That
1058 // way, later passes can recompute register class constraints for inline
1059 // assembly as well as normal instructions.
1060 // Don't do this for tied operands that can use the regclass information
1061 // from the def.
1063 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1064 Flag.setRegClass(RC->getID());
1065 }
1066
1067 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1068 Ops.push_back(Res);
1069
1070 if (Code == InlineAsm::Kind::Clobber) {
1071 // Clobbers should always have a 1:1 mapping with registers, and may
1072 // reference registers that have illegal (e.g. vector) types. Hence, we
1073 // shouldn't try to apply any sort of splitting logic to them.
1074 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1075 "No 1:1 mapping from clobbers to regs?");
1076 Register SP = TLI.getStackPointerRegisterToSaveRestore();
1077 (void)SP;
1078 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1079 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1080 assert(
1081 (Regs[I] != SP ||
1083 "If we clobbered the stack pointer, MFI should know about it.");
1084 }
1085 return;
1086 }
1087
1088 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1089 MVT RegisterVT = RegVTs[Value];
1090 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1091 RegisterVT);
1092 for (unsigned i = 0; i != NumRegs; ++i) {
1093 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1094 Register TheReg = Regs[Reg++];
1095 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1096 }
1097 }
1098}
1099
1103 unsigned I = 0;
1104 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1105 unsigned RegCount = std::get<0>(CountAndVT);
1106 MVT RegisterVT = std::get<1>(CountAndVT);
1107 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1108 for (unsigned E = I + RegCount; I != E; ++I)
1109 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1110 }
1111 return OutVec;
1112}
1113
1115 AssumptionCache *ac, const TargetLibraryInfo *li,
1116 const TargetTransformInfo &TTI) {
1117 BatchAA = aa;
1118 AC = ac;
1119 GFI = gfi;
1120 LibInfo = li;
1121 Context = DAG.getContext();
1122 LPadToCallSiteMap.clear();
1123 this->TTI = &TTI;
1124 SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1125 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1126 *DAG.getMachineFunction().getFunction().getParent());
1127 CanDescribeGlobalAddressInLocationList =
1128 canDescribeGlobalAddressInLocationList(DAG.getMachineFunction());
1129}
1130
1132 NodeMap.clear();
1133 UnusedArgNodeMap.clear();
1134 PendingLoads.clear();
1135 PendingExports.clear();
1136 PendingConstrainedFP.clear();
1137 PendingConstrainedFPStrict.clear();
1138 CurInst = nullptr;
1139 HasTailCall = false;
1140 SDNodeOrder = LowestSDNodeOrder;
1141 StatepointLowering.clear();
1142}
1143
1145 DanglingDebugInfoMap.clear();
1146}
1147
1148// Update DAG root to include dependencies on Pending chains.
1149SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1150 SDValue Root = DAG.getRoot();
1151
1152 if (Pending.empty())
1153 return Root;
1154
1155 // Add current root to PendingChains, unless we already indirectly
1156 // depend on it.
1157 if (Root.getOpcode() != ISD::EntryToken) {
1158 unsigned i = 0, e = Pending.size();
1159 for (; i != e; ++i) {
1160 assert(Pending[i].getNode()->getNumOperands() > 1);
1161 if (Pending[i].getNode()->getOperand(0) == Root)
1162 break; // Don't add the root if we already indirectly depend on it.
1163 }
1164
1165 if (i == e)
1166 Pending.push_back(Root);
1167 }
1168
1169 if (Pending.size() == 1)
1170 Root = Pending[0];
1171 else
1172 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1173
1174 DAG.setRoot(Root);
1175 Pending.clear();
1176 return Root;
1177}
1178
1182
1184 // If the new exception behavior differs from that of the pending
1185 // ones, chain up them and update the root.
1186 switch (EB) {
1189 // Floating-point exceptions produced by such operations are not intended
1190 // to be observed, so the sequence of these operations does not need to be
1191 // preserved.
1192 //
1193 // They however must not be mixed with the instructions that have strict
1194 // exception behavior. Placing an operation with 'ebIgnore' behavior between
1195 // 'ebStrict' operations could distort the observed exception behavior.
1196 if (!PendingConstrainedFPStrict.empty()) {
1197 assert(PendingConstrainedFP.empty());
1198 updateRoot(PendingConstrainedFPStrict);
1199 }
1200 break;
1202 // Floating-point exception produced by these operations may be observed, so
1203 // they must be correctly chained. If trapping on FP exceptions is
1204 // disabled, the exceptions can be observed only by functions that read
1205 // exception flags, like 'llvm.get_fpenv' or 'fetestexcept'. It means that
1206 // the order of operations is not significant between barriers.
1207 //
1208 // If trapping is enabled, each operation becomes an implicit observation
1209 // point, so the operations must be sequenced according their original
1210 // source order.
1211 if (!PendingConstrainedFP.empty()) {
1212 assert(PendingConstrainedFPStrict.empty());
1213 updateRoot(PendingConstrainedFP);
1214 }
1215 // TODO: Add support for trapping-enabled scenarios.
1216 }
1217 return DAG.getRoot();
1218}
1219
1221 // Chain up all pending constrained intrinsics together with all
1222 // pending loads, by simply appending them to PendingLoads and
1223 // then calling getMemoryRoot().
1224 PendingLoads.reserve(PendingLoads.size() +
1225 PendingConstrainedFP.size() +
1226 PendingConstrainedFPStrict.size());
1227 PendingLoads.append(PendingConstrainedFP.begin(),
1228 PendingConstrainedFP.end());
1229 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1230 PendingConstrainedFPStrict.end());
1231 PendingConstrainedFP.clear();
1232 PendingConstrainedFPStrict.clear();
1233 return getMemoryRoot();
1234}
1235
1237 // We need to emit pending fpexcept.strict constrained intrinsics,
1238 // so append them to the PendingExports list.
1239 PendingExports.append(PendingConstrainedFPStrict.begin(),
1240 PendingConstrainedFPStrict.end());
1241 PendingConstrainedFPStrict.clear();
1242 return updateRoot(PendingExports);
1243}
1244
1246 DILocalVariable *Variable,
1248 DebugLoc DL) {
1249 assert(Variable && "Missing variable");
1250
1251 // Check if address has undef value.
1252 if (!Address || isa<UndefValue>(Address) ||
1253 (Address->use_empty() && !isa<Argument>(Address))) {
1254 LLVM_DEBUG(
1255 dbgs()
1256 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1257 return;
1258 }
1259
1260 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1261
1262 SDValue &N = NodeMap[Address];
1263 if (!N.getNode() && isa<Argument>(Address))
1264 // Check unused arguments map.
1265 N = UnusedArgNodeMap[Address];
1266 SDDbgValue *SDV;
1267 if (N.getNode()) {
1268 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1269 Address = BCI->getOperand(0);
1270 // Parameters are handled specially.
1271 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1272 if (IsParameter && FINode) {
1273 // Byval parameter. We have a frame index at this point.
1274 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1275 /*IsIndirect*/ true, DL, SDNodeOrder);
1276 } else if (isa<Argument>(Address)) {
1277 // Address is an argument, so try to emit its dbg value using
1278 // virtual register info from the FuncInfo.ValueMap.
1279 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1280 FuncArgumentDbgValueKind::Declare, N);
1281 return;
1282 } else {
1283 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1284 true, DL, SDNodeOrder);
1285 }
1286 DAG.AddDbgValue(SDV, IsParameter);
1287 } else {
1288 // If Address is an argument then try to emit its dbg value using
1289 // virtual register info from the FuncInfo.ValueMap.
1290 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1291 FuncArgumentDbgValueKind::Declare, N)) {
1292 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1293 << " (could not emit func-arg dbg_value)\n");
1294 }
1295 }
1296}
1297
1299 // Add SDDbgValue nodes for any var locs here. Do so before updating
1300 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1301 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1302 // Add SDDbgValue nodes for any var locs here. Do so before updating
1303 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1304 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1305 It != End; ++It) {
1306 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1307 dropDanglingDebugInfo(Var, It->Expr);
1308 if (It->Values.isKillLocation(It->Expr)) {
1309 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1310 continue;
1311 }
1312 SmallVector<Value *> Values(It->Values.location_ops());
1313 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1314 It->Values.hasArgList())) {
1315 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1317 FnVarLocs->getDILocalVariable(It->VariableID),
1318 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1319 }
1320 }
1321 }
1322
1323 // We must skip DbgVariableRecords if they've already been processed above as
1324 // we have just emitted the debug values resulting from assignment tracking
1325 // analysis, making any existing DbgVariableRecords redundant (and probably
1326 // less correct). We still need to process DbgLabelRecords. This does sink
1327 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1328 // be important as it does so deterministcally and ordering between
1329 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1330 // printing).
1331 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1332 // Is there is any debug-info attached to this instruction, in the form of
1333 // DbgRecord non-instruction debug-info records.
1334 for (DbgRecord &DR : I.getDbgRecordRange()) {
1335 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1336 assert(DLR->getLabel() && "Missing label");
1337 SDDbgLabel *SDV =
1338 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1339 DAG.AddDbgLabel(SDV);
1340 continue;
1341 }
1342
1343 if (SkipDbgVariableRecords)
1344 continue;
1346 DILocalVariable *Variable = DVR.getVariable();
1349
1351 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1352 continue;
1353 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1354 << "\n");
1356 DVR.getDebugLoc());
1357 continue;
1358 }
1359
1360 // A DbgVariableRecord with no locations is a kill location.
1362 if (Values.empty()) {
1364 SDNodeOrder);
1365 continue;
1366 }
1367
1368 // A DbgVariableRecord with an undef or absent location is also a kill
1369 // location.
1370 if (llvm::any_of(Values,
1371 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1373 SDNodeOrder);
1374 continue;
1375 }
1376
1377 bool IsVariadic = DVR.hasArgList();
1378 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1379 SDNodeOrder, IsVariadic)) {
1380 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1381 DVR.getDebugLoc(), SDNodeOrder);
1382 }
1383 }
1384}
1385
1387 visitDbgInfo(I);
1388
1389 // Set up outgoing PHI node register values before emitting the terminator.
1390 if (I.isTerminator()) {
1391 HandlePHINodesInSuccessorBlocks(I.getParent());
1392 }
1393
1394 ++SDNodeOrder;
1395 CurInst = &I;
1396
1397 // Set inserted listener only if required.
1398 bool NodeInserted = false;
1399 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1400 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1401 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);
1402 if (PCSectionsMD || MMRA) {
1403 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1404 DAG, [&](SDNode *) { NodeInserted = true; });
1405 }
1406
1407 visit(I.getOpcode(), I);
1408
1409 if (!I.isTerminator() && !HasTailCall &&
1410 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1412
1413 // Handle metadata.
1414 if (PCSectionsMD || MMRA) {
1415 auto It = NodeMap.find(&I);
1416 if (It != NodeMap.end()) {
1417 if (PCSectionsMD)
1418 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1419 if (MMRA)
1420 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1421 } else if (NodeInserted) {
1422 // This should not happen; if it does, don't let it go unnoticed so we can
1423 // fix it. Relevant visit*() function is probably missing a setValue().
1424 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1425 << I.getModule()->getName() << "]\n";
1426 LLVM_DEBUG(I.dump());
1427 assert(false);
1428 }
1429 }
1430
1431 CurInst = nullptr;
1432}
1433
1434void SelectionDAGBuilder::visitPHI(const PHINode &) {
1435 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1436}
1437
1438void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1439 // Note: this doesn't use InstVisitor, because it has to work with
1440 // ConstantExpr's in addition to instructions.
1441 switch (Opcode) {
1442 default: llvm_unreachable("Unknown instruction type encountered!");
1443 // Build the switch statement using the Instruction.def file.
1444#define HANDLE_INST(NUM, OPCODE, CLASS) \
1445 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1446#include "llvm/IR/Instruction.def"
1447 }
1448}
1449
1451 DILocalVariable *Variable,
1452 DebugLoc DL, unsigned Order,
1455 // For variadic dbg_values we will now insert poison.
1456 // FIXME: We can potentially recover these!
1458 for (const Value *V : Values) {
1459 auto *Poison = PoisonValue::get(V->getType());
1460 Locs.push_back(SDDbgOperand::fromConst(Poison));
1461 }
1462 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1463 /*IsIndirect=*/false, DL, Order,
1464 /*IsVariadic=*/true);
1465 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1466 return true;
1467}
1468
1470 DILocalVariable *Var,
1471 DIExpression *Expr,
1472 bool IsVariadic, DebugLoc DL,
1473 unsigned Order) {
1474 if (IsVariadic) {
1475 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1476 return;
1477 }
1478 // TODO: Dangling debug info will eventually either be resolved or produce
1479 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1480 // between the original dbg.value location and its resolved DBG_VALUE,
1481 // which we should ideally fill with an extra poison DBG_VALUE.
1482 assert(Values.size() == 1);
1483 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1484}
1485
1487 const DIExpression *Expr) {
1488 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1489 DIVariable *DanglingVariable = DDI.getVariable();
1490 DIExpression *DanglingExpr = DDI.getExpression();
1491 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1492 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1493 << printDDI(nullptr, DDI) << "\n");
1494 return true;
1495 }
1496 return false;
1497 };
1498
1499 for (auto &DDIMI : DanglingDebugInfoMap) {
1500 DanglingDebugInfoVector &DDIV = DDIMI.second;
1501
1502 // If debug info is to be dropped, run it through final checks to see
1503 // whether it can be salvaged.
1504 for (auto &DDI : DDIV)
1505 if (isMatchingDbgValue(DDI))
1506 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1507
1508 erase_if(DDIV, isMatchingDbgValue);
1509 }
1510}
1511
1512// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1513// generate the debug data structures now that we've seen its definition.
1515 SDValue Val) {
1516 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1517 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1518 return;
1519
1520 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1521 for (auto &DDI : DDIV) {
1522 DebugLoc DL = DDI.getDebugLoc();
1523 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1524 DILocalVariable *Variable = DDI.getVariable();
1525 DIExpression *Expr = DDI.getExpression();
1526 assert(Variable->isValidLocationForIntrinsic(DL) &&
1527 "Expected inlined-at fields to agree");
1528 SDDbgValue *SDV;
1529 if (Val.getNode()) {
1530 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1531 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1532 // we couldn't resolve it directly when examining the DbgValue intrinsic
1533 // in the first place we should not be more successful here). Unless we
1534 // have some test case that prove this to be correct we should avoid
1535 // calling EmitFuncArgumentDbgValue here.
1536 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1537 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1538 FuncArgumentDbgValueKind::Value, Val)) {
1539 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1540 << printDDI(V, DDI) << "\n");
1541 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1542 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1543 // inserted after the definition of Val when emitting the instructions
1544 // after ISel. An alternative could be to teach
1545 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1546 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1547 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1548 << ValSDNodeOrder << "\n");
1549 SDV = getDbgValue(Val, Variable, Expr, DL,
1550 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1551 DAG.AddDbgValue(SDV, false);
1552 } else
1553 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1554 << printDDI(V, DDI)
1555 << " in EmitFuncArgumentDbgValue\n");
1556 } else {
1557 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1558 << "\n");
1559 auto Poison = PoisonValue::get(V->getType());
1560 auto SDV =
1561 DAG.getConstantDbgValue(Variable, Expr, Poison, DL, DbgSDNodeOrder);
1562 DAG.AddDbgValue(SDV, false);
1563 }
1564 }
1565 DDIV.clear();
1566}
1567
1568/// If \p V is the address of a describable global, possibly displaced by a
1569/// constant, return the global and fold the displacement into location operand
1570/// \p OpIdx of \p Expr. The displacement rides along in the expression rather
1571/// than in the operand, so that it survives into a DBG_INSTR_REF.
1572static const GlobalValue *
1573getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx,
1574 const MachineFunction &MF) {
1575 const auto *C = dyn_cast<Constant>(V);
1576 if (!C)
1577 return nullptr;
1578 int64_t Offset;
1580 if (GV && Offset) {
1583 Expr = DIExpression::appendOpsToArg(Expr, Ops, OpIdx, /*StackValue=*/false);
1584 }
1585 return GV;
1586}
1587
1589 DanglingDebugInfo &DDI) {
1590 // TODO: For the variadic implementation, instead of only checking the fail
1591 // state of `handleDebugValue`, we need know specifically which values were
1592 // invalid, so that we attempt to salvage only those values when processing
1593 // a DIArgList.
1594 const Value *OrigV = V;
1595 DILocalVariable *Var = DDI.getVariable();
1596 DIExpression *Expr = DDI.getExpression();
1597 DebugLoc DL = DDI.getDebugLoc();
1598 unsigned SDOrder = DDI.getSDNodeOrder();
1599
1600 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1601 // that DW_OP_stack_value is desired.
1602 bool StackValue = true;
1603
1604 // handleDebugValue holds out for a register with the address of a global
1605 // that a location list could not name. With no such register forthcoming,
1606 // naming the global still beats dropping the location.
1607 auto HandleGlobalAddress = [&] {
1608 DIExpression *GVExpr = Expr;
1609 const GlobalValue *GV =
1610 getGlobalAddressDbgOperand(V, GVExpr, 0, DAG.getMachineFunction());
1611 if (!GV)
1612 return false;
1613 SDDbgValue *SDV = DAG.getDbgValueList(
1614 Var, GVExpr, SDDbgOperand::fromGlobalAddr(GV), /*Dependencies=*/{},
1615 /*IsIndirect=*/false, DL, SDOrder, /*IsVariadic=*/false);
1616 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1617 return true;
1618 };
1619
1620 // Can this Value can be encoded without any further work?
1621 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false) ||
1622 HandleGlobalAddress())
1623 return;
1624
1625 // Attempt to salvage back through as many instructions as possible. Bail if
1626 // a non-instruction is seen, such as a constant expression or global
1627 // variable. FIXME: Further work could recover those too.
1628 while (isa<Instruction>(V)) {
1629 const Instruction &VAsInst = *cast<const Instruction>(V);
1630 // Temporary "0", awaiting real implementation.
1632 SmallVector<Value *, 4> AdditionalValues;
1633 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1634 Expr->getNumLocationOperands(), Ops,
1635 AdditionalValues);
1636 // If we cannot salvage any further, and haven't yet found a suitable debug
1637 // expression, bail out.
1638 if (!V)
1639 break;
1640
1641 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1642 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1643 // here for variadic dbg_values, remove that condition.
1644 if (!AdditionalValues.empty())
1645 break;
1646
1647 // New value and expr now represent this debuginfo.
1649
1650 // Some kind of simplification occurred: check whether the operand of the
1651 // salvaged debug expression can be encoded in this DAG.
1652 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false) ||
1653 HandleGlobalAddress()) {
1654 LLVM_DEBUG(
1655 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1656 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1657 return;
1658 }
1659 }
1660
1661 // This was the final opportunity to salvage this debug information, and it
1662 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1663 // any earlier variable location.
1664 assert(OrigV && "V shouldn't be null");
1665 auto *Poison = PoisonValue::get(OrigV->getType());
1666 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Poison, DL, SDOrder);
1667 DAG.AddDbgValue(SDV, false);
1668 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1669 << printDDI(OrigV, DDI) << "\n");
1670}
1671
1673 DIExpression *Expr,
1674 DebugLoc DbgLoc,
1675 unsigned Order) {
1676 Value *Poison = PoisonValue::get(Type::getInt1Ty(*Context));
1679 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1680 /*IsVariadic*/ false);
1681}
1682
1684 DILocalVariable *Var,
1685 DIExpression *Expr, DebugLoc DbgLoc,
1686 unsigned Order, bool IsVariadic) {
1687 if (Values.empty())
1688 return true;
1689
1690 // Filter EntryValue locations out early.
1691 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1692 return true;
1693
1694 SmallVector<SDDbgOperand> LocationOps;
1695 SmallVector<SDNode *> Dependencies;
1696 for (const auto &[OpIdx, V] : enumerate(Values)) {
1697 // Constant value.
1700 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1701 continue;
1702 }
1703
1704 // Look through IntToPtr constants.
1705 if (auto *CE = dyn_cast<ConstantExpr>(V))
1706 if (CE->getOpcode() == Instruction::IntToPtr) {
1707 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1708 continue;
1709 }
1710
1711 // The address of a global is a link-time constant, and so is a constant
1712 // displacement from one. A global whose address cannot be described this
1713 // way falls through to be described by whatever materializes it instead.
1714 // So does one that a location list could not name, should the variable
1715 // need one; salvageUnresolvedDbgValue names it if nothing materializes it.
1716 if (CanDescribeGlobalAddressInLocationList)
1718 V, Expr, OpIdx, DAG.getMachineFunction())) {
1720 continue;
1721 }
1722
1723 // If the Value is a frame index, we can create a FrameIndex debug value
1724 // without relying on the DAG at all.
1725 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1726 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1727 if (SI != FuncInfo.StaticAllocaMap.end()) {
1728 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1729 continue;
1730 }
1731 }
1732
1733 // Do not use getValue() in here; we don't want to generate code at
1734 // this point if it hasn't been done yet.
1735 SDValue N = NodeMap[V];
1736 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1737 N = UnusedArgNodeMap[V];
1738
1739 if (N.getNode()) {
1740 // Only emit func arg dbg value for non-variadic dbg.values for now.
1741 if (!IsVariadic &&
1742 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1743 FuncArgumentDbgValueKind::Value, N))
1744 return true;
1745 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1746 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1747 // describe stack slot locations.
1748 //
1749 // Consider "int x = 0; int *px = &x;". There are two kinds of
1750 // interesting debug values here after optimization:
1751 //
1752 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1753 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1754 //
1755 // Both describe the direct values of their associated variables.
1756 Dependencies.push_back(N.getNode());
1757 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1758 continue;
1759 }
1760 LocationOps.emplace_back(
1761 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1762 continue;
1763 }
1764
1765 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1766 // Special rules apply for the first dbg.values of parameter variables in a
1767 // function. Identify them by the fact they reference Argument Values, that
1768 // they're parameters, and they are parameters of the current function. We
1769 // need to let them dangle until they get an SDNode.
1770 bool IsParamOfFunc =
1771 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1772 if (IsParamOfFunc)
1773 return false;
1774
1775 // The value is not used in this block yet (or it would have an SDNode).
1776 // We still want the value to appear for the user if possible -- if it has
1777 // an associated VReg, we can refer to that instead.
1778 auto VMI = FuncInfo.ValueMap.find(V);
1779 if (VMI != FuncInfo.ValueMap.end()) {
1780 Register Reg = VMI->second;
1781 // If this is a PHI node, it may be split up into several MI PHI nodes
1782 // (in FunctionLoweringInfo::set).
1783 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1784 V->getType(), std::nullopt);
1785 if (RFV.occupiesMultipleRegs()) {
1786 // FIXME: We could potentially support variadic dbg_values here.
1787 if (IsVariadic)
1788 return false;
1789 unsigned Offset = 0;
1790 unsigned BitsToDescribe = 0;
1791 if (auto VarSize = Var->getSizeInBits())
1792 BitsToDescribe = *VarSize;
1793 if (auto Fragment = Expr->getFragmentInfo())
1794 BitsToDescribe = Fragment->SizeInBits;
1795 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1796 // Bail out if all bits are described already.
1797 if (Offset >= BitsToDescribe)
1798 break;
1799 // TODO: handle scalable vectors.
1800 unsigned RegisterSize = RegAndSize.second;
1801 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1802 ? BitsToDescribe - Offset
1803 : RegisterSize;
1804 auto FragmentExpr = DIExpression::createFragmentExpression(
1805 Expr, Offset, FragmentSize);
1806 if (!FragmentExpr)
1807 continue;
1808 SDDbgValue *SDV = DAG.getVRegDbgValue(
1809 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);
1810 DAG.AddDbgValue(SDV, false);
1811 Offset += RegisterSize;
1812 }
1813 return true;
1814 }
1815 // We can use simple vreg locations for variadic dbg_values as well.
1816 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1817 continue;
1818 }
1819 // We failed to create a SDDbgOperand for V.
1820 return false;
1821 }
1822
1823 // We have created a SDDbgOperand for each Value in Values.
1824 assert(!LocationOps.empty());
1825 SDDbgValue *SDV =
1826 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1827 /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
1828 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1829 return true;
1830}
1831
1833 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1834 for (auto &Pair : DanglingDebugInfoMap)
1835 for (auto &DDI : Pair.second)
1836 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1838}
1839
1840/// getCopyFromRegs - If there was virtual register allocated for the value V
1841/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1843 auto It = FuncInfo.ValueMap.find(V);
1844 SDValue Result;
1845
1846 if (It != FuncInfo.ValueMap.end()) {
1847 Register InReg = It->second;
1848
1849 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1850 DAG.getDataLayout(), InReg, Ty,
1851 std::nullopt); // This is not an ABI copy.
1852 SDValue Chain = DAG.getEntryNode();
1853 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1854 V);
1855 resolveDanglingDebugInfo(V, Result);
1856 }
1857
1858 return Result;
1859}
1860
1861/// getValue - Return an SDValue for the given Value.
1863 // If we already have an SDValue for this value, use it. It's important
1864 // to do this first, so that we don't create a CopyFromReg if we already
1865 // have a regular SDValue.
1866 SDValue &N = NodeMap[V];
1867 if (N.getNode()) return N;
1868
1869 // If there's a virtual register allocated and initialized for this
1870 // value, use it.
1871 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1872 return copyFromReg;
1873
1874 // Otherwise create a new SDValue and remember it.
1875 SDValue Val = getValueImpl(V);
1876 NodeMap[V] = Val;
1878 return Val;
1879}
1880
1881void SelectionDAGBuilder::setValueToPoison(const Value *V, const SDLoc &dl) {
1882 if (V->getType()->isVoidTy())
1883 return;
1884
1885 SmallVector<EVT, 4> ValueVTs;
1886 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
1887 V->getType(), ValueVTs);
1888 if (ValueVTs.empty())
1889 return;
1890 setValue(V, DAG.getErrorMergeValues(ValueVTs, SDValue(), dl));
1891}
1892
1893/// getNonRegisterValue - Return an SDValue for the given Value, but
1894/// don't look in FuncInfo.ValueMap for a virtual register.
1896 // If we already have an SDValue for this value, use it.
1897 SDValue &N = NodeMap[V];
1898 if (N.getNode()) {
1899 if (isIntOrFPConstant(N)) {
1900 // Remove the debug location from the node as the node is about to be used
1901 // in a location which may differ from the original debug location. This
1902 // is relevant to Constant and ConstantFP nodes because they can appear
1903 // as constant expressions inside PHI nodes.
1904 N->setDebugLoc(DebugLoc());
1905 }
1906 return N;
1907 }
1908
1909 // Otherwise create a new SDValue and remember it.
1910 SDValue Val = getValueImpl(V);
1911 NodeMap[V] = Val;
1913 return Val;
1914}
1915
1916/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1917/// Create an SDValue for the given value.
1919 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1920
1921 if (const Constant *C = dyn_cast<Constant>(V)) {
1922 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1923
1924 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) {
1925 SDLoc DL = getCurSDLoc();
1926
1927 // DAG.getConstant() may attempt to legalise the vector constant which can
1928 // significantly change the combines applied to the DAG. To reduce the
1929 // divergence when enabling ConstantInt based vectors we try to construct
1930 // the DAG in the same way as shufflevector based splats. TODO: The
1931 // divergence sometimes leads to better optimisations. Ideally we should
1932 // prevent DAG.getConstant() from legalising too early but there are some
1933 // degradations preventing this.
1934 if (VT.isScalableVector())
1935 return DAG.getNode(
1936 ISD::SPLAT_VECTOR, DL, VT,
1937 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));
1938 if (VT.isFixedLengthVector())
1939 return DAG.getSplatBuildVector(
1940 VT, DL,
1941 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));
1942 return DAG.getConstant(*CI, DL, VT);
1943 }
1944
1945 if (const ConstantByte *CB = dyn_cast<ConstantByte>(C))
1946 return DAG.getConstant(CB->getValue(), getCurSDLoc(), VT);
1947
1948 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1949 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1950
1951 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
1952 return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT,
1953 getValue(CPA->getPointer()), getValue(CPA->getKey()),
1954 getValue(CPA->getAddrDiscriminator()),
1955 getValue(CPA->getDiscriminator()));
1956 }
1957
1959 return DAG.getConstant(0, getCurSDLoc(), VT);
1960
1961 if (match(C, m_VScale()))
1962 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1963
1964 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1965 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1966
1967 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1968 return isa<PoisonValue>(C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1969
1970 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1971 visit(CE->getOpcode(), *CE);
1972 SDValue N1 = NodeMap[V];
1973 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1974 return N1;
1975 }
1976
1978 SmallVector<SDValue, 4> Constants;
1979 for (const Use &U : C->operands()) {
1980 SDNode *Val = getValue(U).getNode();
1981 // If the operand is an empty aggregate, there are no values.
1982 if (!Val) continue;
1983 // Add each leaf value from the operand to the Constants list
1984 // to form a flattened list of all the values.
1985 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1986 Constants.push_back(SDValue(Val, i));
1987 }
1988
1989 return DAG.getMergeValues(Constants, getCurSDLoc());
1990 }
1991
1992 if (const ConstantDataSequential *CDS =
1995 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1996 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1997 // Add each leaf value from the operand to the Constants list
1998 // to form a flattened list of all the values.
1999 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
2000 Ops.push_back(SDValue(Val, i));
2001 }
2002
2003 if (isa<ArrayType>(CDS->getType()))
2004 return DAG.getMergeValues(Ops, getCurSDLoc());
2005 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);
2006 }
2007
2008 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
2010 "Unknown struct or array constant!");
2011
2012 SmallVector<EVT, 4> ValueVTs;
2013 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
2014 unsigned NumElts = ValueVTs.size();
2015 if (NumElts == 0)
2016 return SDValue(); // empty struct
2017 SmallVector<SDValue, 4> Constants(NumElts);
2018 for (unsigned i = 0; i != NumElts; ++i) {
2019 EVT EltVT = ValueVTs[i];
2020 if (isa<UndefValue>(C))
2021 Constants[i] = DAG.getUNDEF(EltVT);
2022 else if (EltVT.isFloatingPoint())
2023 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
2024 else
2025 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
2026 }
2027
2028 return DAG.getMergeValues(Constants, getCurSDLoc());
2029 }
2030
2031 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
2032 return DAG.getBlockAddress(BA, VT);
2033
2034 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
2035 return getValue(Equiv->getGlobalValue());
2036
2037 if (const auto *NC = dyn_cast<NoCFIValue>(C))
2038 return getValue(NC->getGlobalValue());
2039
2040 if (VT == MVT::aarch64svcount) {
2041 assert(C->isNullValue() && "Can only zero this target type!");
2042 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
2043 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
2044 }
2045
2046 if (VT.isRISCVVectorTuple()) {
2047 assert(C->isNullValue() && "Can only zero this target type!");
2048 return DAG.getNode(
2050 DAG.getNode(
2052 EVT::getVectorVT(*DAG.getContext(), MVT::i8,
2053 VT.getSizeInBits().getKnownMinValue() / 8, true),
2054 DAG.getConstant(0, getCurSDLoc(), MVT::getIntegerVT(8))));
2055 }
2056
2057 if (VT == MVT::externref || VT == MVT::funcref) {
2058 assert(C->isNullValue() && "Can only zero this target type!");
2059 // The zero value of a WebAssembly reference type is the null reference,
2060 // materialized with ref.null.
2061 Intrinsic::ID IID = VT == MVT::externref ? Intrinsic::wasm_ref_null_extern
2062 : Intrinsic::wasm_ref_null_func;
2063 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VT,
2064 DAG.getTargetConstant(IID, getCurSDLoc(), MVT::i32));
2065 }
2066
2067 VectorType *VecTy = cast<VectorType>(V->getType());
2068
2069 // Now that we know the number and type of the elements, get that number of
2070 // elements into the Ops array based on what kind of constant it is.
2071 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
2073 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
2074 for (unsigned i = 0; i != NumElements; ++i)
2075 Ops.push_back(getValue(CV->getOperand(i)));
2076
2077 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);
2078 }
2079
2081 EVT EltVT =
2082 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
2083
2084 SDValue Op;
2085 if (EltVT.isFloatingPoint())
2086 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
2087 else
2088 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
2089
2090 return DAG.getSplat(VT, getCurSDLoc(), Op);
2091 }
2092
2093 llvm_unreachable("Unknown vector constant");
2094 }
2095
2096 // If this is a static alloca, generate it as the frameindex instead of
2097 // computation.
2098 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
2099 auto SI = FuncInfo.StaticAllocaMap.find(AI);
2100 if (SI != FuncInfo.StaticAllocaMap.end())
2101 return DAG.getFrameIndex(
2102 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
2103 }
2104
2105 // If this is an instruction which fast-isel has deferred, select it now.
2106 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
2107 Register InReg = FuncInfo.InitializeRegForValue(Inst);
2108 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
2109 Inst->getType(), std::nullopt);
2110 SDValue Chain = DAG.getEntryNode();
2111 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
2112 }
2113
2114 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
2115 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
2116
2117 if (const auto *BB = dyn_cast<BasicBlock>(V))
2118 return DAG.getBasicBlock(FuncInfo.getMBB(BB));
2119
2120 llvm_unreachable("Can't get register for value!");
2121}
2122
2123void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
2125 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
2126 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
2127 bool IsSEH = isAsynchronousEHPersonality(Pers);
2128 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
2129 if (IsSEH) {
2130 // For SEH, EHCont Guard needs to know that this catchpad is a target.
2131 CatchPadMBB->setIsEHContTarget(true);
2133 } else
2134 CatchPadMBB->setIsEHScopeEntry();
2135 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
2136 if (IsMSVCCXX || IsCoreCLR)
2137 CatchPadMBB->setIsEHFuncletEntry();
2138}
2139
2140void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
2141 // Update machine-CFG edge.
2142 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());
2143 FuncInfo.MBB->addSuccessor(TargetMBB);
2144
2145 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
2146 bool IsSEH = isAsynchronousEHPersonality(Pers);
2147 if (IsSEH) {
2148 // If this is not a fall-through branch or optimizations are switched off,
2149 // emit the branch.
2150 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
2151 TM.getOptLevel() == CodeGenOptLevel::None)
2152 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2153 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
2154 return;
2155 }
2156
2157 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2158 TargetMBB->setIsEHContTarget(true);
2159 DAG.getMachineFunction().setHasEHContTarget(true);
2160
2161 // Figure out the funclet membership for the catchret's successor.
2162 // This will be used by the FuncletLayout pass to determine how to order the
2163 // BB's.
2164 // A 'catchret' returns to the outer scope's color.
2165 Value *ParentPad = I.getCatchSwitchParentPad();
2166 const BasicBlock *SuccessorColor;
2167 if (isa<ConstantTokenNone>(ParentPad))
2168 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2169 else
2170 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
2171 assert(SuccessorColor && "No parent funclet for catchret!");
2172 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(SuccessorColor);
2173 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2174
2175 // Create the terminator node.
2176 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
2177 getControlRoot(), DAG.getBasicBlock(TargetMBB),
2178 DAG.getBasicBlock(SuccessorColorMBB));
2179 DAG.setRoot(Ret);
2180}
2181
2182void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2183 // Don't emit any special code for the cleanuppad instruction. It just marks
2184 // the start of an EH scope/funclet.
2185 FuncInfo.MBB->setIsEHScopeEntry();
2186 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
2187 if (Pers != EHPersonality::Wasm_CXX && Pers != EHPersonality::Wasm_D) {
2188 FuncInfo.MBB->setIsEHFuncletEntry();
2189 FuncInfo.MBB->setIsCleanupFuncletEntry();
2190 }
2191}
2192
2193/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2194/// many places it could ultimately go. In the IR, we have a single unwind
2195/// destination, but in the machine CFG, we enumerate all the possible blocks.
2196/// This function skips over imaginary basic blocks that hold catchswitch
2197/// instructions, and finds all the "real" machine
2198/// basic block destinations. As those destinations may not be successors of
2199/// EHPadBB, here we also calculate the edge probability to those destinations.
2200/// The passed-in Prob is the edge probability to EHPadBB.
2202 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2203 BranchProbability Prob,
2204 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2205 &UnwindDests) {
2206 EHPersonality Personality =
2208 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2209 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2210 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2211 bool IsWasmD = Personality == EHPersonality::Wasm_D;
2212 bool IsSEH = isAsynchronousEHPersonality(Personality);
2213
2214 while (EHPadBB) {
2216 BasicBlock *NewEHPadBB = nullptr;
2217 if (isa<LandingPadInst>(Pad)) {
2218 // Stop on landingpads. They are not funclets.
2219 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2220 break;
2221 } else if (isa<CleanupPadInst>(Pad)) {
2222 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2223 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2224 // which always catches an exception.
2225 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2226 UnwindDests.back().first->setIsEHScopeEntry();
2227 // In Wasm, EH scopes are not funclets
2228 if (!IsWasmCXX && !IsWasmD)
2229 UnwindDests.back().first->setIsEHFuncletEntry();
2230 break;
2231 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2232 // Add the catchpad handlers to the possible destinations.
2233 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2234 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2235 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2236 if (IsMSVCCXX || IsCoreCLR)
2237 UnwindDests.back().first->setIsEHFuncletEntry();
2238 if (!IsSEH)
2239 UnwindDests.back().first->setIsEHScopeEntry();
2240 }
2241 NewEHPadBB = CatchSwitch->getUnwindDest();
2242 } else {
2243 continue;
2244 }
2245
2246 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2247 if (BPI && NewEHPadBB)
2248 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2249 EHPadBB = NewEHPadBB;
2250 }
2251}
2252
2253void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2254 // Update successor info.
2256 auto UnwindDest = I.getUnwindDest();
2257 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2258 BranchProbability UnwindDestProb =
2259 (BPI && UnwindDest)
2260 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2262 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2263 for (auto &UnwindDest : UnwindDests) {
2264 UnwindDest.first->setIsEHPad();
2265 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2266 }
2267 FuncInfo.MBB->normalizeSuccProbs();
2268
2269 // Create the terminator node.
2270 MachineBasicBlock *CleanupPadMBB =
2271 FuncInfo.getMBB(I.getCleanupPad()->getParent());
2272 SDValue Ret = DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other,
2273 getControlRoot(), DAG.getBasicBlock(CleanupPadMBB));
2274 DAG.setRoot(Ret);
2275}
2276
2277void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2278 report_fatal_error("visitCatchSwitch not yet implemented!");
2279}
2280
2281void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2282 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2283 auto &DL = DAG.getDataLayout();
2284 SDValue Chain = getControlRoot();
2287
2288 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2289 // lower
2290 //
2291 // %val = call <ty> @llvm.experimental.deoptimize()
2292 // ret <ty> %val
2293 //
2294 // differently.
2295 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2297 return;
2298 }
2299
2300 if (!FuncInfo.CanLowerReturn) {
2301 Register DemoteReg = FuncInfo.DemoteRegister;
2302
2303 // Emit a store of the return value through the virtual register.
2304 // Leave Outs empty so that LowerReturn won't try to load return
2305 // registers the usual way.
2306 MVT PtrValueVT = TLI.getPointerTy(DL, DL.getAllocaAddrSpace());
2307 SDValue RetPtr =
2308 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVT);
2309 Type *RetTy = I.getOperand(0)->getType();
2310 Align BaseAlign = DL.getPrefTypeAlign(RetTy);
2311 RetPtr =
2312 TLI.annotateStackObjectPointer(RetPtr, DAG, getCurSDLoc(), BaseAlign);
2313 SDValue RetOp = getValue(I.getOperand(0));
2314
2315 SmallVector<EVT, 4> ValueVTs, MemVTs;
2316 SmallVector<uint64_t, 4> Offsets;
2317 ComputeValueVTs(TLI, DL, RetTy, ValueVTs, &MemVTs, &Offsets, 0);
2318 unsigned NumValues = ValueVTs.size();
2319
2320 SmallVector<SDValue, 4> Chains(NumValues);
2321 for (unsigned i = 0; i != NumValues; ++i) {
2322 // An aggregate return value cannot wrap around the address space, so
2323 // offsets to its parts don't wrap either.
2324 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
2325 TypeSize::getFixed(Offsets[i]));
2326
2327 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2328 if (MemVTs[i] != ValueVTs[i])
2329 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2330 Chains[i] = DAG.getStore(
2331 Chain, getCurSDLoc(), Val,
2332 // FIXME: better loc info would be nice.
2333 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
2334 commonAlignment(BaseAlign, Offsets[i]));
2335 }
2336
2337 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
2338 MVT::Other, Chains);
2339 } else if (I.getNumOperands() != 0) {
2341 ComputeValueTypes(DL, I.getOperand(0)->getType(), Types);
2342 unsigned NumValues = Types.size();
2343 if (NumValues) {
2344 SDValue RetOp = getValue(I.getOperand(0));
2345
2346 const Function *F = I.getParent()->getParent();
2347
2349 I.getOperand(0)->getType(), F->getCallingConv(),
2350 /*IsVarArg*/ false, DL);
2351
2353 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2355 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2357
2358 LLVMContext &Context = F->getContext();
2359 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2360
2361 for (unsigned j = 0; j != NumValues; ++j) {
2362 EVT VT = TLI.getValueType(DL, Types[j]);
2363
2364 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2366
2367 CallingConv::ID CC = F->getCallingConv();
2368
2370 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2373 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2374 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2375
2376 // 'inreg' on function refers to return value
2377 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2378 if (RetInReg)
2379 Flags.setInReg();
2380
2381 if (I.getOperand(0)->getType()->isPointerTy()) {
2382 Flags.setPointer();
2383 Flags.setPointerAddrSpace(
2384 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2385 }
2386
2387 if (NeedsRegBlock) {
2388 Flags.setInConsecutiveRegs();
2389 if (j == NumValues - 1)
2390 Flags.setInConsecutiveRegsLast();
2391 }
2392
2393 // Propagate extension type if any
2395 Flags.setSExt();
2396 else if (ExtendKind == ISD::ZERO_EXTEND)
2397 Flags.setZExt();
2398 else if (F->getAttributes().hasRetAttr(Attribute::NoExt))
2399 Flags.setNoExt();
2400
2401 for (unsigned i = 0; i < NumParts; ++i) {
2402 Outs.push_back(ISD::OutputArg(Flags,
2403 Parts[i].getValueType().getSimpleVT(),
2404 VT, Types[j], 0, 0));
2405 OutVals.push_back(Parts[i]);
2406 }
2407 }
2408 }
2409 }
2410
2411 // Push in swifterror virtual register as the last element of Outs. This makes
2412 // sure swifterror virtual register will be returned in the swifterror
2413 // physical register.
2414 const Function *F = I.getParent()->getParent();
2415 if (TLI.supportSwiftError() &&
2416 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2417 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2418 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2419 Flags.setSwiftError();
2420 Outs.push_back(ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
2421 /*argvt=*/EVT(TLI.getPointerTy(DL)),
2422 PointerType::getUnqual(*DAG.getContext()),
2423 /*origidx=*/1, /*partOffs=*/0));
2424 // Create SDNode for the swifterror virtual register.
2425 OutVals.push_back(
2426 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2427 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2428 EVT(TLI.getPointerTy(DL))));
2429 }
2430
2431 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2432 CallingConv::ID CallConv =
2433 DAG.getMachineFunction().getFunction().getCallingConv();
2434 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2435 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2436
2437 // Verify that the target's LowerReturn behaved as expected.
2438 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2439 "LowerReturn didn't return a valid chain!");
2440
2441 // Update the DAG with the new chain value resulting from return lowering.
2442 DAG.setRoot(Chain);
2443}
2444
2445/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2446/// created for it, emit nodes to copy the value into the virtual
2447/// registers.
2449 // Skip empty types
2450 if (V->getType()->isEmptyTy())
2451 return;
2452
2453 auto VMI = FuncInfo.ValueMap.find(V);
2454 if (VMI != FuncInfo.ValueMap.end()) {
2455 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2456 "Unused value assigned virtual registers!");
2457 CopyValueToVirtualRegister(V, VMI->second);
2458 }
2459}
2460
2461/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2462/// the current basic block, add it to ValueMap now so that we'll get a
2463/// CopyTo/FromReg.
2465 // No need to export constants.
2466 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2467
2468 // Already exported?
2469 if (FuncInfo.isExportedInst(V)) return;
2470
2471 Register Reg = FuncInfo.InitializeRegForValue(V);
2473}
2474
2476 const BasicBlock *FromBB) {
2477 // The operands of the setcc have to be in this block. We don't know
2478 // how to export them from some other block.
2479 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2480 // Can export from current BB.
2481 if (VI->getParent() == FromBB)
2482 return true;
2483
2484 // Is already exported, noop.
2485 return FuncInfo.isExportedInst(V);
2486 }
2487
2488 // If this is an argument, we can export it if the BB is the entry block or
2489 // if it is already exported.
2490 if (isa<Argument>(V)) {
2491 if (FromBB->isEntryBlock())
2492 return true;
2493
2494 // Otherwise, can only export this if it is already exported.
2495 return FuncInfo.isExportedInst(V);
2496 }
2497
2498 // Otherwise, constants can always be exported.
2499 return true;
2500}
2501
2502/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2504SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2505 const MachineBasicBlock *Dst) const {
2507 const BasicBlock *SrcBB = Src->getBasicBlock();
2508 const BasicBlock *DstBB = Dst->getBasicBlock();
2509 if (!BPI) {
2510 // If BPI is not available, set the default probability as 1 / N, where N is
2511 // the number of successors.
2512 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2513 return BranchProbability(1, SuccSize);
2514 }
2515 return BPI->getEdgeProbability(SrcBB, DstBB);
2516}
2517
2518void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2519 MachineBasicBlock *Dst,
2520 BranchProbability Prob) {
2521 if (!FuncInfo.BPI)
2522 Src->addSuccessorWithoutProb(Dst);
2523 else {
2524 if (Prob.isUnknown())
2525 Prob = getEdgeProbability(Src, Dst);
2526 Src->addSuccessor(Dst, Prob);
2527 }
2528}
2529
2530static bool InBlock(const Value *V, const BasicBlock *BB) {
2531 if (const Instruction *I = dyn_cast<Instruction>(V))
2532 return I->getParent() == BB;
2533 return true;
2534}
2535
2536/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2537/// This function emits a branch and is used at the leaves of an OR or an
2538/// AND operator tree.
2539void
2542 MachineBasicBlock *FBB,
2543 MachineBasicBlock *CurBB,
2544 MachineBasicBlock *SwitchBB,
2545 BranchProbability TProb,
2546 BranchProbability FProb,
2547 bool InvertCond) {
2548 const BasicBlock *BB = CurBB->getBasicBlock();
2549
2550 // If the leaf of the tree is a comparison, merge the condition into
2551 // the caseblock.
2552 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2553 // The operands of the cmp have to be in this block. We don't know
2554 // how to export them from some other block. If this is the first block
2555 // of the sequence, no exporting is needed.
2556 if (CurBB == SwitchBB ||
2557 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2558 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2559 ISD::CondCode Condition;
2560 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2561 ICmpInst::Predicate Pred =
2562 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2563 Condition = getICmpCondCode(Pred);
2564 } else {
2565 const FCmpInst *FC = cast<FCmpInst>(Cond);
2566 FCmpInst::Predicate Pred =
2567 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2568 Condition = getFCmpCondCode(Pred);
2569 if (FC->hasNoNaNs() ||
2570 (isKnownNeverNaN(FC->getOperand(0),
2571 SimplifyQuery(DAG.getDataLayout(), FC)) &&
2572 isKnownNeverNaN(FC->getOperand(1),
2573 SimplifyQuery(DAG.getDataLayout(), FC))))
2574 Condition = getFCmpCodeWithoutNaN(Condition);
2575 }
2576
2577 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2578 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2579 SL->SwitchCases.push_back(CB);
2580 return;
2581 }
2582 }
2583
2584 // Create a CaseBlock record representing this branch.
2585 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2586 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2587 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2588 SL->SwitchCases.push_back(CB);
2589}
2590
2591// Collect dependencies on V recursively. This is used for the cost analysis in
2592// `shouldKeepJumpConditionsTogether`.
2596 unsigned Depth = 0) {
2597 // Return false if we have an incomplete count.
2599 return false;
2600
2601 auto *I = dyn_cast<Instruction>(V);
2602 if (I == nullptr)
2603 return true;
2604
2605 if (Necessary != nullptr) {
2606 // This instruction is necessary for the other side of the condition so
2607 // don't count it.
2608 if (Necessary->contains(I))
2609 return true;
2610 }
2611
2612 // Already added this dep.
2613 if (!Deps->try_emplace(I, false).second)
2614 return true;
2615
2616 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2617 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2618 Depth + 1))
2619 return false;
2620 return true;
2621}
2622
2625 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2627 if (Params.BaseCost < 0)
2628 return false;
2629
2630 // Baseline cost.
2631 InstructionCost CostThresh = Params.BaseCost;
2632
2633 BranchProbabilityInfo *BPI = nullptr;
2634 if (Params.LikelyBias || Params.UnlikelyBias)
2635 BPI = FuncInfo.BPI;
2636 if (BPI != nullptr) {
2637 // See if we are either likely to get an early out or compute both lhs/rhs
2638 // of the condition.
2639 BasicBlock *IfTrue = I.getSuccessor(0);
2640 BasicBlock *IfFalse = I.getSuccessor(1);
2642 std::swap(IfTrue, IfFalse);
2643
2644 std::optional<bool> Likely;
2645 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2646 Likely = true;
2647 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2648 Likely = false;
2649
2650 if (Likely) {
2651 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2652 // Its likely we will have to compute both lhs and rhs of condition
2653 CostThresh += Params.LikelyBias;
2654 else {
2655 if (Params.UnlikelyBias < 0)
2656 return false;
2657 // Its likely we will get an early out.
2658 CostThresh -= Params.UnlikelyBias;
2659 }
2660 }
2661 }
2662
2663 if (CostThresh <= 0)
2664 return false;
2665
2666 // Collect "all" instructions that lhs condition is dependent on.
2667 // Use map for stable iteration (to avoid non-determanism of iteration of
2668 // SmallPtrSet). The `bool` value is just a dummy.
2670 collectInstructionDeps(&LhsDeps, Lhs);
2671 // Collect "all" instructions that rhs condition is dependent on AND are
2672 // dependencies of lhs. This gives us an estimate on which instructions we
2673 // stand to save by splitting the condition.
2674 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2675 return false;
2676 // Add the compare instruction itself unless its a dependency on the LHS.
2677 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2678 if (!LhsDeps.contains(RhsI))
2679 RhsDeps.try_emplace(RhsI, false);
2680
2681 InstructionCost CostOfIncluding = 0;
2682 // See if this instruction will need to computed independently of whether RHS
2683 // is.
2684 Value *BrCond = I.getCondition();
2685 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2686 for (const auto *U : Ins->users()) {
2687 // If user is independent of RHS calculation we don't need to count it.
2688 if (auto *UIns = dyn_cast<Instruction>(U))
2689 if (UIns != BrCond && !RhsDeps.contains(UIns))
2690 return false;
2691 }
2692 return true;
2693 };
2694
2695 // Prune instructions from RHS Deps that are dependencies of unrelated
2696 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2697 // arbitrary and just meant to cap the how much time we spend in the pruning
2698 // loop. Its highly unlikely to come into affect.
2699 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2700 // Stop after a certain point. No incorrectness from including too many
2701 // instructions.
2702 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2703 const Instruction *ToDrop = nullptr;
2704 for (const auto &InsPair : RhsDeps) {
2705 if (!ShouldCountInsn(InsPair.first)) {
2706 ToDrop = InsPair.first;
2707 break;
2708 }
2709 }
2710 if (ToDrop == nullptr)
2711 break;
2712 RhsDeps.erase(ToDrop);
2713 }
2714
2715 for (const auto &InsPair : RhsDeps) {
2716 // Finally accumulate latency that we can only attribute to computing the
2717 // RHS condition. Use latency because we are essentially trying to calculate
2718 // the cost of the dependency chain.
2719 // Possible TODO: We could try to estimate ILP and make this more precise.
2720 CostOfIncluding += TTI->getInstructionCost(
2721 InsPair.first, TargetTransformInfo::TCK_Latency);
2722
2723 if (CostOfIncluding > CostThresh)
2724 return false;
2725 }
2726 return true;
2727}
2728
2731 MachineBasicBlock *FBB,
2732 MachineBasicBlock *CurBB,
2733 MachineBasicBlock *SwitchBB,
2735 BranchProbability TProb,
2736 BranchProbability FProb,
2737 bool InvertCond) {
2738 // Skip over not part of the tree and remember to invert op and operands at
2739 // next level.
2740 Value *NotCond;
2741 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2742 InBlock(NotCond, CurBB->getBasicBlock())) {
2743 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2744 !InvertCond);
2745 return;
2746 }
2747
2749 const Value *BOpOp0, *BOpOp1;
2750 // Compute the effective opcode for Cond, taking into account whether it needs
2751 // to be inverted, e.g.
2752 // and (not (or A, B)), C
2753 // gets lowered as
2754 // and (and (not A, not B), C)
2756 if (BOp) {
2757 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2758 ? Instruction::And
2759 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2760 ? Instruction::Or
2762 if (InvertCond) {
2763 if (BOpc == Instruction::And)
2764 BOpc = Instruction::Or;
2765 else if (BOpc == Instruction::Or)
2766 BOpc = Instruction::And;
2767 }
2768 }
2769
2770 // If this node is not part of the or/and tree, emit it as a branch.
2771 // Note that all nodes in the tree should have same opcode.
2772 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2773 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2774 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2775 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2776 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2777 TProb, FProb, InvertCond);
2778 return;
2779 }
2780
2781 // Create TmpBB after CurBB.
2782 MachineFunction::iterator BBI(CurBB);
2783 MachineFunction &MF = DAG.getMachineFunction();
2785 CurBB->getParent()->insert(++BBI, TmpBB);
2786
2787 if (Opc == Instruction::Or) {
2788 // Codegen X | Y as:
2789 // BB1:
2790 // jmp_if_X TBB
2791 // jmp TmpBB
2792 // TmpBB:
2793 // jmp_if_Y TBB
2794 // jmp FBB
2795 //
2796
2797 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2798 // The requirement is that
2799 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2800 // = TrueProb for original BB.
2801 // Assuming the original probabilities are A and B, one choice is to set
2802 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2803 // A/(1+B) and 2B/(1+B). This choice assumes that
2804 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2805 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2806 // TmpBB, but the math is more complicated.
2807
2808 auto NewTrueProb = TProb / 2;
2809 auto NewFalseProb = TProb / 2 + FProb;
2810 // Emit the LHS condition.
2811 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2812 NewFalseProb, InvertCond);
2813
2814 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2815 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2817 // Emit the RHS condition into TmpBB.
2818 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2819 Probs[1], InvertCond);
2820 } else {
2821 assert(Opc == Instruction::And && "Unknown merge op!");
2822 // Codegen X & Y as:
2823 // BB1:
2824 // jmp_if_X TmpBB
2825 // jmp FBB
2826 // TmpBB:
2827 // jmp_if_Y TBB
2828 // jmp FBB
2829 //
2830 // This requires creation of TmpBB after CurBB.
2831
2832 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2833 // The requirement is that
2834 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2835 // = FalseProb for original BB.
2836 // Assuming the original probabilities are A and B, one choice is to set
2837 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2838 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2839 // TrueProb for BB1 * FalseProb for TmpBB.
2840
2841 auto NewTrueProb = TProb + FProb / 2;
2842 auto NewFalseProb = FProb / 2;
2843 // Emit the LHS condition.
2844 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2845 NewFalseProb, InvertCond);
2846
2847 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2848 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2850 // Emit the RHS condition into TmpBB.
2851 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2852 Probs[1], InvertCond);
2853 }
2854}
2855
2856/// If the set of cases should be emitted as a series of branches, return true.
2857/// If we should emit this as a bunch of and/or'd together conditions, return
2858/// false.
2859bool
2860SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2861 if (Cases.size() != 2) return true;
2862
2863 // If this is two comparisons of the same values or'd or and'd together, they
2864 // will get folded into a single comparison, so don't emit two blocks.
2865 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2866 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2867 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2868 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2869 return false;
2870 }
2871
2872 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2873 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2874 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2875 Cases[0].CC == Cases[1].CC &&
2876 isa<Constant>(Cases[0].CmpRHS) &&
2877 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2878 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2879 return false;
2880 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2881 return false;
2882 }
2883
2884 return true;
2885}
2886
2887void SelectionDAGBuilder::visitUncondBr(const UncondBrInst &I) {
2889
2890 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2891
2892 // Update machine-CFG edges.
2893 BrMBB->addSuccessor(Succ0MBB);
2894
2895 // If this is not a fall-through branch or optimizations are switched off,
2896 // emit the branch.
2897 if (Succ0MBB != NextBlock(BrMBB) ||
2899 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2900 DAG.getBasicBlock(Succ0MBB));
2901 setValue(&I, Br);
2902 DAG.setRoot(Br);
2903 }
2904}
2905
2906void SelectionDAGBuilder::visitCondBr(const CondBrInst &I) {
2907 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2908
2909 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2910
2911 // If this condition is one of the special cases we handle, do special stuff
2912 // now.
2913 const Value *CondVal = I.getCondition();
2914 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(I.getSuccessor(1));
2915
2916 // If this is a series of conditions that are or'd or and'd together, emit
2917 // this as a sequence of branches instead of setcc's with and/or operations.
2918 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2919 // unpredictable branches, and vector extracts because those jumps are likely
2920 // expensive for any target), this should improve performance.
2921 // For example, instead of something like:
2922 // cmp A, B
2923 // C = seteq
2924 // cmp D, E
2925 // F = setle
2926 // or C, F
2927 // jnz foo
2928 // Emit:
2929 // cmp A, B
2930 // je foo
2931 // cmp D, E
2932 // jle foo
2933 bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
2934 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2935 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2936 BOp->hasOneUse() && !IsUnpredictable) {
2937 Value *Vec;
2938 const Value *BOp0, *BOp1;
2940 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2941 Opcode = Instruction::And;
2942 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2943 Opcode = Instruction::Or;
2944
2945 if (Opcode &&
2946 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2947 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2949 FuncInfo, I, Opcode, BOp0, BOp1,
2950 DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2951 Opcode, BOp0, BOp1, FuncInfo.Fn))) {
2952 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2953 getEdgeProbability(BrMBB, Succ0MBB),
2954 getEdgeProbability(BrMBB, Succ1MBB),
2955 /*InvertCond=*/false);
2956 // If the compares in later blocks need to use values not currently
2957 // exported from this block, export them now. This block should always
2958 // be the first entry.
2959 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2960
2961 // Allow some cases to be rejected.
2962 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2963 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2964 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2965 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2966 }
2967
2968 // Emit the branch for this block.
2969 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2970 SL->SwitchCases.erase(SL->SwitchCases.begin());
2971 return;
2972 }
2973
2974 // Okay, we decided not to do this, remove any inserted MBB's and clear
2975 // SwitchCases.
2976 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2977 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2978
2979 SL->SwitchCases.clear();
2980 }
2981 }
2982
2983 // Create a CaseBlock record representing this branch.
2984 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2985 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2987 IsUnpredictable);
2988
2989 // Use visitSwitchCase to actually insert the fast branch sequence for this
2990 // cond branch.
2991 visitSwitchCase(CB, BrMBB);
2992}
2993
2994/// visitSwitchCase - Emits the necessary code to represent a single node in
2995/// the binary search tree resulting from lowering a switch instruction.
2997 MachineBasicBlock *SwitchBB) {
2998 SDValue Cond;
2999 SDValue CondLHS = getValue(CB.CmpLHS);
3000 SDLoc dl = CB.DL;
3001
3002 if (CB.CC == ISD::SETTRUE) {
3003 // Branch or fall through to TrueBB.
3004 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
3005 SwitchBB->normalizeSuccProbs();
3006 if (CB.TrueBB != NextBlock(SwitchBB)) {
3007 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
3008 DAG.getBasicBlock(CB.TrueBB)));
3009 }
3010 return;
3011 }
3012
3013 auto &TLI = DAG.getTargetLoweringInfo();
3014 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
3015
3016 // Build the setcc now.
3017 if (!CB.CmpMHS) {
3018 // Fold "(X == true)" to X and "(X == false)" to !X to
3019 // handle common cases produced by branch lowering.
3020 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
3021 CB.CC == ISD::SETEQ)
3022 Cond = CondLHS;
3023 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
3024 CB.CC == ISD::SETEQ) {
3025 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
3026 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
3027 } else {
3028 SDValue CondRHS = getValue(CB.CmpRHS);
3029
3030 // If a pointer's DAG type is larger than its memory type then the DAG
3031 // values are zero-extended. This breaks signed comparisons so truncate
3032 // back to the underlying type before doing the compare.
3033 if (CondLHS.getValueType() != MemVT) {
3034 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
3035 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
3036 }
3037 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
3038 }
3039 } else {
3040 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
3041
3042 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
3043 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
3044
3045 SDValue CmpOp = getValue(CB.CmpMHS);
3046 EVT VT = CmpOp.getValueType();
3047
3048 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
3049 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
3050 ISD::SETLE);
3051 } else {
3052 SDValue SUB = DAG.getNode(ISD::SUB, dl,
3053 VT, CmpOp, DAG.getConstant(Low, dl, VT));
3054 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
3055 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
3056 }
3057 }
3058
3059 // Update successor info
3060 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
3061 // TrueBB and FalseBB are always different unless the incoming IR is
3062 // degenerate. This only happens when running llc on weird IR.
3063 if (CB.TrueBB != CB.FalseBB)
3064 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
3065 SwitchBB->normalizeSuccProbs();
3066
3067 // If the lhs block is the next block, invert the condition so that we can
3068 // fall through to the lhs instead of the rhs block.
3069 if (CB.TrueBB == NextBlock(SwitchBB)) {
3070 std::swap(CB.TrueBB, CB.FalseBB);
3071 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
3072 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
3073 }
3074
3075 SDNodeFlags Flags;
3077 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
3078 Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
3079
3080 setValue(CurInst, BrCond);
3081
3082 // Insert the false branch. Do this even if it's a fall through branch,
3083 // this makes it easier to do DAG optimizations which require inverting
3084 // the branch condition.
3085 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3086 DAG.getBasicBlock(CB.FalseBB));
3087
3088 DAG.setRoot(BrCond);
3089}
3090
3091/// visitJumpTable - Emit JumpTable node in the current MBB
3093 // Emit the code for the jump table
3094 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3095 assert(JT.Reg && "Should lower JT Header first!");
3096 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DAG.getDataLayout());
3097 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
3098 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
3099 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
3100 Index.getValue(1), Table, Index);
3101 DAG.setRoot(BrJumpTable);
3102}
3103
3104/// visitJumpTableHeader - This function emits necessary code to produce index
3105/// in the JumpTable from switch case.
3107 JumpTableHeader &JTH,
3108 MachineBasicBlock *SwitchBB) {
3109 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3110 const SDLoc &dl = *JT.SL;
3111
3112 // Subtract the lowest switch case value from the value being switched on.
3113 SDValue SwitchOp = getValue(JTH.SValue);
3114 EVT VT = SwitchOp.getValueType();
3115 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
3116 DAG.getConstant(JTH.First, dl, VT));
3117
3118 // The SDNode we just created, which holds the value being switched on minus
3119 // the smallest case value, needs to be copied to a virtual register so it
3120 // can be used as an index into the jump table in a subsequent basic block.
3121 // This value may be smaller or larger than the target's pointer type, and
3122 // therefore require extension or truncating.
3123 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3124 SwitchOp =
3125 DAG.getZExtOrTrunc(Sub, dl, TLI.getJumpTableRegTy(DAG.getDataLayout()));
3126
3127 Register JumpTableReg =
3128 FuncInfo.CreateReg(TLI.getJumpTableRegTy(DAG.getDataLayout()));
3129 SDValue CopyTo =
3130 DAG.getCopyToReg(getControlRoot(), dl, JumpTableReg, SwitchOp);
3131 JT.Reg = JumpTableReg;
3132
3133 if (!JTH.FallthroughUnreachable) {
3134 // Emit the range check for the jump table, and branch to the default block
3135 // for the switch statement if the value being switched on exceeds the
3136 // largest case in the switch.
3137 SDValue CMP = DAG.getSetCC(
3138 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3139 Sub.getValueType()),
3140 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3141
3142 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3143 MVT::Other, CopyTo, CMP,
3144 DAG.getBasicBlock(JT.Default));
3145
3146 // Avoid emitting unnecessary branches to the next block.
3147 if (JT.MBB != NextBlock(SwitchBB))
3148 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3149 DAG.getBasicBlock(JT.MBB));
3150
3151 DAG.setRoot(BrCond);
3152 } else {
3153 // Avoid emitting unnecessary branches to the next block.
3154 if (JT.MBB != NextBlock(SwitchBB))
3155 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3156 DAG.getBasicBlock(JT.MBB)));
3157 else
3158 DAG.setRoot(CopyTo);
3159 }
3160}
3161
3162/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3163/// variable if there exists one.
3165 SDValue &Chain) {
3167 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3168 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3170 Value *Global =
3171 TLI.getSDagStackGuard(*MF.getFunction().getParent(), DAG.getLibcalls());
3173 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3174 if (Global) {
3175 MachinePointerInfo MPInfo(Global);
3179 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
3180 DAG.setNodeMemRefs(Node, {MemRef});
3181 }
3182 if (PtrTy != PtrMemTy)
3183 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3184 return SDValue(Node, 0);
3185}
3186
3187/// Codegen a new tail for a stack protector check ParentMBB which has had its
3188/// tail spliced into a stack protector check success bb.
3189///
3190/// For a high level explanation of how this fits into the stack protector
3191/// generation see the comment on the declaration of class
3192/// StackProtectorDescriptor.
3194 MachineBasicBlock *ParentBB) {
3195
3196 // First create the loads to the guard/stack slot for the comparison.
3197 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3198 auto &DL = DAG.getDataLayout();
3199 EVT PtrTy = TLI.getFrameIndexTy(DL);
3200 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
3201
3202 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3203 int FI = MFI.getStackProtectorIndex();
3204
3205 SDValue Guard;
3206 SDLoc dl = getCurSDLoc();
3207 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3208 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3209 Align Align = DL.getPrefTypeAlign(
3210 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
3211
3212 // Generate code to load the content of the guard slot.
3213 SDValue GuardVal = DAG.getLoad(
3214 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3215 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
3217
3218 // If cookie mixing is enabled, unmix the stored GuardVal to get back the
3219 // original cookie for comparison. The prologue stored (FP - Cookie) or
3220 // (FP XOR Cookie), so we apply the same operation again to unmix:
3221 // FP - (FP - Cookie) = Cookie, or (FP XOR Cookie) XOR FP = Cookie.
3222 if (TLI.useStackGuardMixFP())
3223 GuardVal = TLI.emitStackGuardMixFP(DAG, GuardVal, dl);
3224
3225 // If we're using function-based instrumentation, call the guard check
3226 // function
3228 // Get the guard check function from the target and verify it exists since
3229 // we're using function-based instrumentation
3230 const Function *GuardCheckFn =
3231 TLI.getSSPStackGuardCheck(M, DAG.getLibcalls());
3232 assert(GuardCheckFn && "Guard check function is null");
3233
3234 // The target provides a guard check function to validate the guard value.
3235 // Generate a call to that function with the content of the guard slot as
3236 // argument.
3237 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3238 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3239
3241 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
3242 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3243 Entry.IsInReg = true;
3244 Args.push_back(Entry);
3245
3248 .setChain(DAG.getEntryNode())
3249 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3250 getValue(GuardCheckFn), std::move(Args));
3251
3252 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3253 DAG.setRoot(Result.second);
3254 return;
3255 }
3256
3257 // Load the fresh guard value for comparison.
3258 // For targets that mix the cookie in LOAD_STACK_GUARD expansion, we need to
3259 // load directly without using LOAD_STACK_GUARD to avoid unwanted mixing.
3260 SDValue Chain = DAG.getEntryNode();
3261 if (TLI.useStackGuardMixFP()) {
3262 // Mixing targets: load cookie directly to avoid mixing in LOAD_STACK_GUARD
3263 if (const Value *IRGuard = TLI.getSDagStackGuard(M, DAG.getLibcalls())) {
3264 SDValue GuardPtr = getValue(IRGuard);
3265 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3266 MachinePointerInfo(IRGuard, 0), Align,
3268 } else {
3269 LLVMContext &Ctx = *DAG.getContext();
3270 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
3271 Guard = DAG.getPOISON(PtrMemTy);
3272 }
3273 } else {
3274 // Non-mixing targets: use LOAD_STACK_GUARD or direct load as usual
3275 if (TLI.useLoadStackGuardNode(M)) {
3276 Guard = getLoadStackGuard(DAG, dl, Chain);
3277 } else {
3278 if (const Value *IRGuard = TLI.getSDagStackGuard(M, DAG.getLibcalls())) {
3279 SDValue GuardPtr = getValue(IRGuard);
3280 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3281 MachinePointerInfo(IRGuard, 0), Align,
3283 } else {
3284 LLVMContext &Ctx = *DAG.getContext();
3285 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
3286 Guard = DAG.getPOISON(PtrMemTy);
3287 }
3288 }
3289 }
3290
3291 // Now both Guard (fresh cookie) and GuardVal (unmixed from stored value)
3292 // contain unmixed cookie values that can be compared directly.
3293
3294 // Perform the comparison via a getsetcc.
3295 SDValue Cmp = DAG.getSetCC(
3296 dl, TLI.getSetCCResultType(DL, *DAG.getContext(), Guard.getValueType()),
3297 Guard, GuardVal, ISD::SETNE);
3298
3299 // If the guard/stackslot do not equal, branch to failure MBB.
3300 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
3301 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3302 // Otherwise branch to success MBB.
3303 SDValue Br = DAG.getNode(ISD::BR, dl,
3304 MVT::Other, BrCond,
3305 DAG.getBasicBlock(SPD.getSuccessMBB()));
3306
3307 DAG.setRoot(Br);
3308}
3309
3310/// Codegen the failure basic block for a stack protector check.
3311///
3312/// A failure stack protector machine basic block consists simply of a call to
3313/// __stack_chk_fail().
3314///
3315/// For a high level explanation of how this fits into the stack protector
3316/// generation see the comment on the declaration of class
3317/// StackProtectorDescriptor.
3320
3321 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3322 MachineBasicBlock *ParentBB = SPD.getParentMBB();
3323 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3324 SDValue Chain;
3325
3326 // For -Oz builds with a guard check function, we use function-based
3327 // instrumentation. Otherwise, if we have a guard check function, we call it
3328 // in the failure block.
3329 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M, DAG.getLibcalls());
3330 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3331 // First create the loads to the guard/stack slot for the comparison.
3332 auto &DL = DAG.getDataLayout();
3333 EVT PtrTy = TLI.getFrameIndexTy(DL);
3334 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
3335
3336 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3337 int FI = MFI.getStackProtectorIndex();
3338
3339 SDLoc dl = getCurSDLoc();
3340 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3341 Align Align = DL.getPrefTypeAlign(
3342 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
3343
3344 // Generate code to load the content of the guard slot.
3345 SDValue GuardVal = DAG.getLoad(
3346 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3347 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
3349
3350 if (TLI.useStackGuardMixFP())
3351 GuardVal = TLI.emitStackGuardMixFP(DAG, GuardVal, dl);
3352
3353 // The target provides a guard check function to validate the guard value.
3354 // Generate a call to that function with the content of the guard slot as
3355 // argument.
3356 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3357 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3358
3360 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
3361 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3362 Entry.IsInReg = true;
3363 Args.push_back(Entry);
3364
3367 .setChain(DAG.getEntryNode())
3368 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3369 getValue(GuardCheckFn), std::move(Args));
3370
3371 Chain = TLI.LowerCallTo(CLI).second;
3372 } else {
3374 CallOptions.setDiscardResult(true);
3375 Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3376 {}, CallOptions, getCurSDLoc())
3377 .second;
3378 }
3379
3380 // Emit a trap instruction if we are required to do so.
3381 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3382 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3383 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3384
3385 DAG.setRoot(Chain);
3386}
3387
3388/// visitBitTestHeader - This function emits necessary code to produce value
3389/// suitable for "bit tests"
3391 MachineBasicBlock *SwitchBB) {
3392 SDLoc dl = getCurSDLoc();
3393
3394 // Subtract the minimum value.
3395 SDValue SwitchOp = getValue(B.SValue);
3396 EVT VT = SwitchOp.getValueType();
3397 SDValue RangeSub =
3398 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3399
3400 // Determine the type of the test operands.
3401 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3402 bool UsePtrType = false;
3403 if (!TLI.isTypeLegal(VT)) {
3404 UsePtrType = true;
3405 } else {
3406 for (const BitTestCase &Case : B.Cases)
3407 if (!isUIntN(VT.getSizeInBits(), Case.Mask)) {
3408 // Switch table case range are encoded into series of masks.
3409 // Just use pointer type, it's guaranteed to fit.
3410 UsePtrType = true;
3411 break;
3412 }
3413 }
3414 SDValue Sub = RangeSub;
3415 if (UsePtrType) {
3416 VT = TLI.getPointerTy(DAG.getDataLayout());
3417 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3418 }
3419
3420 B.RegVT = VT.getSimpleVT();
3421 B.Reg = FuncInfo.CreateReg(B.RegVT);
3422 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3423
3424 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3425
3426 if (!B.FallthroughUnreachable)
3427 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3428 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3429 SwitchBB->normalizeSuccProbs();
3430
3431 SDValue Root = CopyTo;
3432 if (!B.FallthroughUnreachable) {
3433 // Conditional branch to the default block.
3434 SDValue RangeCmp = DAG.getSetCC(dl,
3435 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3436 RangeSub.getValueType()),
3437 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3438 ISD::SETUGT);
3439
3440 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3441 DAG.getBasicBlock(B.Default));
3442 }
3443
3444 // Avoid emitting unnecessary branches to the next block.
3445 if (MBB != NextBlock(SwitchBB))
3446 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3447
3448 DAG.setRoot(Root);
3449}
3450
3451/// visitBitTestCase - this function produces one "bit test"
3453 MachineBasicBlock *NextMBB,
3454 BranchProbability BranchProbToNext,
3455 Register Reg, BitTestCase &B,
3456 MachineBasicBlock *SwitchBB) {
3457 SDLoc dl = getCurSDLoc();
3458 MVT VT = BB.RegVT;
3459 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3460 SDValue Cmp;
3461 unsigned PopCount = llvm::popcount(B.Mask);
3462 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3463 if (PopCount == 1) {
3464 // Testing for a single bit; just compare the shift count with what it
3465 // would need to be to shift a 1 bit in that position.
3466 Cmp = DAG.getSetCC(
3467 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3468 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3469 ISD::SETEQ);
3470 } else if (PopCount == BB.Range) {
3471 // There is only one zero bit in the range, test for it directly.
3472 Cmp = DAG.getSetCC(
3473 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3474 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3475 } else {
3476 // Make desired shift
3477 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3478 DAG.getConstant(1, dl, VT), ShiftOp);
3479
3480 // Emit bit tests and jumps
3481 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3482 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3483 Cmp = DAG.getSetCC(
3484 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3485 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3486 }
3487
3488 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3489 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3490 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3491 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3492 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3493 // one as they are relative probabilities (and thus work more like weights),
3494 // and hence we need to normalize them to let the sum of them become one.
3495 SwitchBB->normalizeSuccProbs();
3496
3497 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3498 MVT::Other, getControlRoot(),
3499 Cmp, DAG.getBasicBlock(B.TargetBB));
3500
3501 // Avoid emitting unnecessary branches to the next block.
3502 if (NextMBB != NextBlock(SwitchBB))
3503 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3504 DAG.getBasicBlock(NextMBB));
3505
3506 DAG.setRoot(BrAnd);
3507}
3508
3509void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3510 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3511
3512 // Retrieve successors. Look through artificial IR level blocks like
3513 // catchswitch for successors.
3514 MachineBasicBlock *Return = FuncInfo.getMBB(I.getSuccessor(0));
3515 const BasicBlock *EHPadBB = I.getSuccessor(1);
3516 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(EHPadBB);
3517
3518 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3519 // have to do anything here to lower funclet bundles.
3520 failForInvalidBundles(I, "invokes",
3526
3527 const Value *Callee(I.getCalledOperand());
3528 const Function *Fn = dyn_cast<Function>(Callee);
3529 if (isa<InlineAsm>(Callee))
3530 visitInlineAsm(I, EHPadBB);
3531 else if (Fn && Fn->isIntrinsic()) {
3532 switch (Fn->getIntrinsicID()) {
3533 default:
3534 llvm_unreachable("Cannot invoke this intrinsic");
3535 case Intrinsic::donothing:
3536 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3537 case Intrinsic::seh_try_begin:
3538 case Intrinsic::seh_scope_begin:
3539 case Intrinsic::seh_try_end:
3540 case Intrinsic::seh_scope_end:
3541 if (EHPadMBB)
3542 // a block referenced by EH table
3543 // so dtor-funclet not removed by opts
3544 EHPadMBB->setMachineBlockAddressTaken();
3545 break;
3546 case Intrinsic::experimental_patchpoint_void:
3547 case Intrinsic::experimental_patchpoint:
3548 visitPatchpoint(I, EHPadBB);
3549 break;
3550 case Intrinsic::experimental_gc_statepoint:
3552 break;
3553 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3554 // but these intrinsics are special because they can be invoked, so we
3555 // manually lower it to a DAG node here.
3556 case Intrinsic::wasm_throw: {
3558 std::array<SDValue, 4> Ops = {
3559 getControlRoot(), // inchain for the terminator node
3560 DAG.getTargetConstant(Intrinsic::wasm_throw, getCurSDLoc(),
3561 TLI.getPointerTy(DAG.getDataLayout())),
3562 getValue(I.getArgOperand(0)), // tag
3563 getValue(I.getArgOperand(1)) // thrown value
3564 };
3565 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3566 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3567 break;
3568 }
3569 case Intrinsic::wasm_rethrow: {
3570 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3571 std::array<SDValue, 2> Ops = {
3572 getControlRoot(), // inchain for the terminator node
3573 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3574 TLI.getPointerTy(DAG.getDataLayout()))};
3575 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3576 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3577 break;
3578 }
3579 // EH labels mark it as an invoke site, like a call.
3580 case Intrinsic::bpf_unwind_exit: {
3581 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3582 MCSymbol *BeginLabel;
3583 std::array<SDValue, 2> Ops = {
3584 lowerStartEH(getControlRoot(), EHPadBB, BeginLabel),
3585 DAG.getTargetConstant(Intrinsic::bpf_unwind_exit, getCurSDLoc(),
3586 TLI.getPointerTy(DAG.getDataLayout()))};
3587 SDValue Chain =
3588 DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), MVT::Other, Ops);
3589 DAG.setRoot(lowerEndEH(Chain, &I, EHPadBB, BeginLabel));
3590 break;
3591 }
3592 }
3593 } else if (I.hasDeoptState()) {
3594 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3595 // Eventually we will support lowering the @llvm.experimental.deoptimize
3596 // intrinsic, and right now there are no plans to support other intrinsics
3597 // with deopt state.
3598 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3599 } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
3601 } else {
3602 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3603 }
3604
3605 // If the value of the invoke is used outside of its defining block, make it
3606 // available as a virtual register.
3607 // We already took care of the exported value for the statepoint instruction
3608 // during call to the LowerStatepoint.
3609 if (!isa<GCStatepointInst>(I)) {
3611 }
3612
3614 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3615 BranchProbability EHPadBBProb =
3616 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3618 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3619
3620 // Update successor info.
3621 addSuccessorWithProb(InvokeMBB, Return);
3622 for (auto &UnwindDest : UnwindDests) {
3623 UnwindDest.first->setIsEHPad();
3624 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3625 }
3626 InvokeMBB->normalizeSuccProbs();
3627
3628 // Drop into normal successor.
3629 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
3630 DAG.getBasicBlock(Return)));
3631}
3632
3633/// The intrinsics currently supported by callbr are implicit control flow
3634/// intrinsics such as amdgcn.kill.
3635/// - they should be called (no "dontcall-" attributes)
3636/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
3637/// - they do not need custom argument handling (no
3638/// TLI.CollectTargetIntrinsicOperands())
3639void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
3640#ifndef NDEBUG
3642 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3643 Infos, I, DAG.getMachineFunction(), I.getIntrinsicID());
3644 assert(Infos.empty() && "Intrinsic touches memory");
3645#endif
3646
3647 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
3648
3650 getTargetIntrinsicOperands(I, HasChain, OnlyLoad);
3651 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
3652
3653 // Create the node.
3654 SDValue Result =
3655 getTargetNonMemIntrinsicNode(*I.getType(), HasChain, Ops, VTs);
3656 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
3657
3658 setValue(&I, Result);
3659}
3660
3661void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3662 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3663
3664 if (I.isInlineAsm()) {
3665 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3666 // have to do anything here to lower funclet bundles.
3667 failForInvalidBundles(I, "callbrs",
3669 visitInlineAsm(I);
3670 } else {
3671 assert(!I.hasOperandBundles() &&
3672 "Can't have operand bundles for intrinsics");
3673 visitCallBrIntrinsic(I);
3674 }
3676
3677 // Retrieve successors.
3678 SmallPtrSet<BasicBlock *, 8> Dests;
3679 Dests.insert(I.getDefaultDest());
3680 MachineBasicBlock *Return = FuncInfo.getMBB(I.getDefaultDest());
3681
3682 // Update successor info.
3683 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3684 // TODO: For most of the cases where there is an intrinsic callbr, we're
3685 // having exactly one indirect target, which will be unreachable. As soon as
3686 // this changes, we might need to enhance
3687 // Target->setIsInlineAsmBrIndirectTarget or add something similar for
3688 // intrinsic indirect branches.
3689 if (I.isInlineAsm()) {
3690 for (BasicBlock *Dest : I.getIndirectDests()) {
3691 MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
3692 Target->setIsInlineAsmBrIndirectTarget();
3693 // If we introduce a type of asm goto statement that is permitted to use
3694 // an indirect call instruction to jump to its labels, then we should add
3695 // a call to Target->setMachineBlockAddressTaken() here, to mark the
3696 // target block as requiring a BTI.
3697
3698 Target->setLabelMustBeEmitted();
3699 // Don't add duplicate machine successors.
3700 if (Dests.insert(Dest).second)
3701 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3702 }
3703 }
3704 CallBrMBB->normalizeSuccProbs();
3705
3706 // Drop into default successor.
3707 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3708 MVT::Other, getControlRoot(),
3709 DAG.getBasicBlock(Return)));
3710}
3711
3712void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3713 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3714}
3715
3716void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3717 assert(FuncInfo.MBB->isEHPad() &&
3718 "Call to landingpad not in landing pad!");
3719
3720 // If there aren't registers to copy the values into (e.g., during SjLj
3721 // exceptions), then don't bother to create these DAG nodes.
3722 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3723 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3724 if (TLI.getExceptionPointerRegister(FuncInfo.ExceptionModel, PersonalityFn) ==
3725 0 &&
3727 PersonalityFn) == 0)
3728 return;
3729
3730 // If landingpad's return type is token type, we don't create DAG nodes
3731 // for its exception pointer and selector value. The extraction of exception
3732 // pointer or selector value from token type landingpads is not currently
3733 // supported.
3734 if (LP.getType()->isTokenTy())
3735 return;
3736
3737 // LangRef leaves the result type target-specific, so diagnose types this
3738 // lowering cannot represent instead of asserting.
3739 SDLoc dl = getCurSDLoc();
3741 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
3742 *LP.getFunction(),
3743 "landingpad result type must be a struct of an exception pointer and "
3744 "an integer selector",
3745 dl.getDebugLoc()));
3746 setValueToPoison(&LP, dl);
3747 return;
3748 }
3749
3750 SmallVector<EVT, 2> ValueVTs;
3751 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3752 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3753
3754 // Get the two live-in registers as SDValues. The physregs have already been
3755 // copied into virtual registers.
3756 SDValue Ops[2];
3757 if (FuncInfo.ExceptionPointerVirtReg) {
3758 Ops[0] = DAG.getZExtOrTrunc(
3759 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3760 FuncInfo.ExceptionPointerVirtReg,
3761 TLI.getPointerTy(DAG.getDataLayout())),
3762 dl, ValueVTs[0]);
3763 } else {
3764 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3765 }
3766 Ops[1] = DAG.getZExtOrTrunc(
3767 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3768 FuncInfo.ExceptionSelectorVirtReg,
3769 TLI.getPointerTy(DAG.getDataLayout())),
3770 dl, ValueVTs[1]);
3771
3772 // Merge into one.
3773 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3774 DAG.getVTList(ValueVTs), Ops);
3775 setValue(&LP, Res);
3776}
3777
3780 // Update JTCases.
3781 for (JumpTableBlock &JTB : SL->JTCases)
3782 if (JTB.first.HeaderBB == First)
3783 JTB.first.HeaderBB = Last;
3784
3785 // Update BitTestCases.
3786 for (BitTestBlock &BTB : SL->BitTestCases)
3787 if (BTB.Parent == First)
3788 BTB.Parent = Last;
3789}
3790
3791void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3792 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3793
3794 // Update machine-CFG edges with unique successors.
3796 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3797 BasicBlock *BB = I.getSuccessor(i);
3798 bool Inserted = Done.insert(BB).second;
3799 if (!Inserted)
3800 continue;
3801
3802 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3803 addSuccessorWithProb(IndirectBrMBB, Succ);
3804 }
3805 IndirectBrMBB->normalizeSuccProbs();
3806
3808 MVT::Other, getControlRoot(),
3809 getValue(I.getAddress())));
3810}
3811
3812void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3813 if (!I.shouldLowerToTrap(DAG.getTarget().Options.TrapUnreachable,
3814 DAG.getTarget().Options.NoTrapAfterNoreturn))
3815 return;
3816
3817 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3818}
3819
3820void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3821 SDNodeFlags Flags;
3822 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3823 Flags.copyFMF(*FPOp);
3824
3825 SDValue Op = getValue(I.getOperand(0));
3826 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3827 Op, Flags);
3828 setValue(&I, UnNodeValue);
3829}
3830
3831void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3832 SDNodeFlags Flags;
3833 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3834 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3835 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3836 }
3837 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3838 Flags.setExact(ExactOp->isExact());
3839 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3840 Flags.setDisjoint(DisjointOp->isDisjoint());
3841 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3842 Flags.copyFMF(*FPOp);
3843
3844 SDValue Op1 = getValue(I.getOperand(0));
3845 SDValue Op2 = getValue(I.getOperand(1));
3846 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3847 Op1, Op2, Flags);
3848 setValue(&I, BinNodeValue);
3849}
3850
3851void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3852 SDValue Op1 = getValue(I.getOperand(0));
3853 SDValue Op2 = getValue(I.getOperand(1));
3854
3855 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3856 Op1.getValueType(), DAG.getDataLayout());
3857
3858 // Coerce the shift amount to the right type if we can. This exposes the
3859 // truncate or zext to optimization early.
3860 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3862 "Unexpected shift type");
3863 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3864 }
3865
3866 bool nuw = false;
3867 bool nsw = false;
3868 bool exact = false;
3869
3870 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3871
3872 if (const OverflowingBinaryOperator *OFBinOp =
3874 nuw = OFBinOp->hasNoUnsignedWrap();
3875 nsw = OFBinOp->hasNoSignedWrap();
3876 }
3877 if (const PossiblyExactOperator *ExactOp =
3879 exact = ExactOp->isExact();
3880 }
3881 SDNodeFlags Flags;
3882 Flags.setExact(exact);
3883 Flags.setNoSignedWrap(nsw);
3884 Flags.setNoUnsignedWrap(nuw);
3885 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3886 Flags);
3887 setValue(&I, Res);
3888}
3889
3890void SelectionDAGBuilder::visitSDiv(const User &I) {
3891 SDValue Op1 = getValue(I.getOperand(0));
3892 SDValue Op2 = getValue(I.getOperand(1));
3893
3894 SDNodeFlags Flags;
3895 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3896 cast<PossiblyExactOperator>(&I)->isExact());
3897 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3898 Op2, Flags));
3899}
3900
3901void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3902 ICmpInst::Predicate predicate = I.getPredicate();
3903 SDValue Op1 = getValue(I.getOperand(0));
3904 SDValue Op2 = getValue(I.getOperand(1));
3905 ISD::CondCode Opcode = getICmpCondCode(predicate);
3906
3907 auto &TLI = DAG.getTargetLoweringInfo();
3908 EVT MemVT =
3909 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3910
3911 // If a pointer's DAG type is larger than its memory type then the DAG values
3912 // are zero-extended. This breaks signed comparisons so truncate back to the
3913 // underlying type before doing the compare.
3914 if (Op1.getValueType() != MemVT) {
3915 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3916 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3917 }
3918
3919 SDNodeFlags Flags;
3920 Flags.setSameSign(I.hasSameSign());
3921
3922 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3923 I.getType());
3924 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode,
3925 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3926}
3927
3928void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3929 FCmpInst::Predicate predicate = I.getPredicate();
3930 SDValue Op1 = getValue(I.getOperand(0));
3931 SDValue Op2 = getValue(I.getOperand(1));
3932
3933 ISD::CondCode Condition = getFCmpCondCode(predicate);
3934 auto *FPMO = cast<FPMathOperator>(&I);
3935 if (FPMO->hasNoNaNs() ||
3936 (DAG.isKnownNeverNaN(Op1) && DAG.isKnownNeverNaN(Op2)))
3937 Condition = getFCmpCodeWithoutNaN(Condition);
3938
3939 SDNodeFlags Flags;
3940 Flags.copyFMF(*FPMO);
3941
3942 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3943 I.getType());
3944 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition,
3945 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3946}
3947
3948// Check if the condition of the select has one use or two users that are both
3949// selects with the same condition.
3950static bool hasOnlySelectUsers(const Value *Cond) {
3951 return llvm::all_of(Cond->users(), [](const Value *V) {
3952 return isa<SelectInst>(V);
3953 });
3954}
3955
3956void SelectionDAGBuilder::visitSelect(const User &I) {
3957 SmallVector<EVT, 4> ValueVTs;
3958 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3959 ValueVTs);
3960 unsigned NumValues = ValueVTs.size();
3961 if (NumValues == 0) return;
3962
3964 SDValue Cond = getValue(I.getOperand(0));
3965 SDValue LHSVal = getValue(I.getOperand(1));
3966 SDValue RHSVal = getValue(I.getOperand(2));
3967 SmallVector<SDValue, 1> BaseOps(1, Cond);
3969 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3970
3971 bool IsUnaryAbs = false;
3972 bool Negate = false;
3973
3974 SDNodeFlags Flags;
3975 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3976 Flags.copyFMF(*FPOp);
3977
3978 Flags.setUnpredictable(
3979 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3980
3981 // Min/max matching is only viable if all output VTs are the same.
3982 if (all_equal(ValueVTs)) {
3983 EVT VT = ValueVTs[0];
3984 LLVMContext &Ctx = *DAG.getContext();
3985 auto &TLI = DAG.getTargetLoweringInfo();
3986
3987 // We care about the legality of the operation after it has been type
3988 // legalized.
3990 VT = TLI.getTypeToTransformTo(Ctx, VT);
3991
3992 // If the vselect is legal, assume we want to leave this as a vector setcc +
3993 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3994 // min/max is legal on the scalar type.
3995 bool UseScalarMinMax = VT.isVector() &&
3997
3998 // ValueTracking's select pattern matching does not account for -0.0,
3999 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
4000 // -0.0 is less than +0.0.
4001 const Value *LHS, *RHS;
4002 auto SPR = matchSelectPattern(&I, LHS, RHS);
4004 switch (SPR.Flavor) {
4005 case SPF_UMAX: Opc = ISD::UMAX; break;
4006 case SPF_UMIN: Opc = ISD::UMIN; break;
4007 case SPF_SMAX: Opc = ISD::SMAX; break;
4008 case SPF_SMIN: Opc = ISD::SMIN; break;
4009 case SPF_FMINNUM:
4011 break;
4012
4013 switch (SPR.NaNBehavior) {
4014 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4015 case SPNB_RETURNS_ANY:
4016 case SPNB_RETURNS_NAN:
4017 break;
4018 case SPNB_RETURNS_OTHER:
4020 Flags.setNoSignedZeros(true);
4021 break;
4022 }
4023 break;
4024 case SPF_FMAXNUM:
4026 break;
4027
4028 switch (SPR.NaNBehavior) {
4029 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4030 case SPNB_RETURNS_NAN:
4031 case SPNB_RETURNS_ANY:
4032 break;
4033 case SPNB_RETURNS_OTHER:
4035 Flags.setNoSignedZeros(true);
4036 break;
4037 }
4038 break;
4039 case SPF_NABS:
4040 Negate = true;
4041 [[fallthrough]];
4042 case SPF_ABS:
4043 IsUnaryAbs = true;
4044 Opc = ISD::ABS;
4045 break;
4046 default: break;
4047 }
4048
4049 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
4051 (UseScalarMinMax &&
4053 // If the underlying comparison instruction is used by any other
4054 // instruction, the consumed instructions won't be destroyed, so it is
4055 // not profitable to convert to a min/max.
4057 OpCode = Opc;
4058 LHSVal = getValue(LHS);
4059 RHSVal = getValue(RHS);
4060 BaseOps.clear();
4061 }
4062
4063 if (IsUnaryAbs) {
4064 OpCode = Opc;
4065 LHSVal = getValue(LHS);
4066 BaseOps.clear();
4067 }
4068 }
4069
4070 if (IsUnaryAbs) {
4071 for (unsigned i = 0; i != NumValues; ++i) {
4072 SDLoc dl = getCurSDLoc();
4073 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
4074 Values[i] =
4075 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
4076 if (Negate)
4077 Values[i] = DAG.getNegative(Values[i], dl, VT);
4078 }
4079 } else {
4080 for (unsigned i = 0; i != NumValues; ++i) {
4081 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
4082 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
4083 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
4084 Values[i] = DAG.getNode(
4085 OpCode, getCurSDLoc(),
4086 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
4087 }
4088 }
4089
4091 DAG.getVTList(ValueVTs), Values));
4092}
4093
4094void SelectionDAGBuilder::visitTrunc(const User &I) {
4095 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
4096 SDValue N = getValue(I.getOperand(0));
4097 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4098 I.getType());
4099 SDNodeFlags Flags;
4100 if (auto *Trunc = dyn_cast<TruncInst>(&I)) {
4101 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
4102 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
4103 }
4104
4105 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N, Flags));
4106}
4107
4108void SelectionDAGBuilder::visitZExt(const User &I) {
4109 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4110 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
4111 SDValue N = getValue(I.getOperand(0));
4112 auto &TLI = DAG.getTargetLoweringInfo();
4113 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4114
4115 SDNodeFlags Flags;
4116 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
4117 Flags.setNonNeg(PNI->hasNonNeg());
4118
4119 // Eagerly use nonneg information to canonicalize towards sign_extend if
4120 // that is the target's preference.
4121 // TODO: Let the target do this later.
4122 if (Flags.hasNonNeg() &&
4123 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
4124 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
4125 return;
4126 }
4127
4128 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
4129}
4130
4131void SelectionDAGBuilder::visitSExt(const User &I) {
4132 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4133 // SExt also can't be a cast to bool for same reason. So, nothing much to do
4134 SDValue N = getValue(I.getOperand(0));
4135 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4136 I.getType());
4137 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
4138}
4139
4140void SelectionDAGBuilder::visitFPTrunc(const User &I) {
4141 // FPTrunc is never a no-op cast, no need to check
4142 SDValue N = getValue(I.getOperand(0));
4143 SDLoc dl = getCurSDLoc();
4144 SDNodeFlags Flags;
4145 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
4146 Flags.copyFMF(*FPOp);
4147 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4148 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4149 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
4150 DAG.getTargetConstant(
4151 0, dl, TLI.getPointerTy(DAG.getDataLayout())),
4152 Flags));
4153}
4154
4155void SelectionDAGBuilder::visitFPExt(const User &I) {
4156 // FPExt is never a no-op cast, no need to check
4157 SDValue N = getValue(I.getOperand(0));
4158 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4159 I.getType());
4160 SDNodeFlags Flags;
4161 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
4162 Flags.copyFMF(*FPOp);
4163 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N, Flags));
4164}
4165
4166void SelectionDAGBuilder::visitFPToUI(const User &I) {
4167 // FPToUI is never a no-op cast, no need to check
4168 SDValue N = getValue(I.getOperand(0));
4169 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4170 I.getType());
4171 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
4172}
4173
4174void SelectionDAGBuilder::visitFPToSI(const User &I) {
4175 // FPToSI is never a no-op cast, no need to check
4176 SDValue N = getValue(I.getOperand(0));
4177 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4178 I.getType());
4179 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
4180}
4181
4182void SelectionDAGBuilder::visitUIToFP(const User &I) {
4183 // UIToFP is never a no-op cast, no need to check
4184 SDValue N = getValue(I.getOperand(0));
4185 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4186 I.getType());
4187 SDNodeFlags Flags;
4188 Flags.setNonNeg(cast<PossiblyNonNegInst>(&I)->hasNonNeg());
4189 Flags.copyFMF(*cast<FPMathOperator>(&I));
4190
4191 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
4192}
4193
4194void SelectionDAGBuilder::visitSIToFP(const User &I) {
4195 // SIToFP is never a no-op cast, no need to check
4196 SDValue N = getValue(I.getOperand(0));
4197 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4198 I.getType());
4199 SDNodeFlags Flags;
4200 Flags.copyFMF(*cast<FPMathOperator>(&I));
4201
4202 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
4203}
4204
4205void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
4206 SDValue N = getValue(I.getOperand(0));
4207 // By definition the type of the ptrtoaddr must be equal to the address type.
4208 const auto &TLI = DAG.getTargetLoweringInfo();
4209 EVT AddrVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4210 // The address width must be smaller or equal to the pointer representation
4211 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).
4212 N = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), AddrVT, N);
4213 setValue(&I, N);
4214}
4215
4216void SelectionDAGBuilder::visitPtrToInt(const User &I) {
4217 // What to do depends on the size of the integer and the size of the pointer.
4218 // We can either truncate, zero extend, or no-op, accordingly.
4219 SDValue N = getValue(I.getOperand(0));
4220 auto &TLI = DAG.getTargetLoweringInfo();
4221 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4222 I.getType());
4223 EVT PtrMemVT =
4224 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
4225 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
4226 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
4227 setValue(&I, N);
4228}
4229
4230void SelectionDAGBuilder::visitIntToPtr(const User &I) {
4231 // What to do depends on the size of the integer and the size of the pointer.
4232 // We can either truncate, zero extend, or no-op, accordingly.
4233 SDValue N = getValue(I.getOperand(0));
4234 auto &TLI = DAG.getTargetLoweringInfo();
4235 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4236 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4237 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
4238 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
4239 setValue(&I, N);
4240}
4241
4242void SelectionDAGBuilder::visitBitCast(const User &I) {
4243 SDValue N = getValue(I.getOperand(0));
4244 SDLoc dl = getCurSDLoc();
4245 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4246 I.getType());
4247
4248 // BitCast assures us that source and destination are the same size so this is
4249 // either a BITCAST or a no-op.
4250 if (DestVT != N.getValueType())
4251 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
4252 DestVT, N)); // convert types.
4253 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
4254 // might fold any kind of constant expression to an integer constant and that
4255 // is not what we are looking for. Only recognize a bitcast of a genuine
4256 // constant integer as an opaque constant.
4257 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
4258 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
4259 /*isOpaque*/true));
4260 else
4261 setValue(&I, N); // noop cast.
4262}
4263
4264void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
4265 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4266 const Value *SV = I.getOperand(0);
4267 SDValue N = getValue(SV);
4268 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4269
4270 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
4271 unsigned DestAS = I.getType()->getPointerAddressSpace();
4272
4273 if (!TM.isNoopAddrSpaceCast(DAG.getDataLayout(), SrcAS, DestAS)) {
4274 SDNodeFlags Flags;
4275 if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(&I))
4276 Flags.setNonNull(ASC->hasNonNull());
4277 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS, Flags);
4278 }
4279
4280 setValue(&I, N);
4281}
4282
4283void SelectionDAGBuilder::visitInsertElement(const User &I) {
4284 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4285 SDValue InVec = getValue(I.getOperand(0));
4286 SDValue InVal = getValue(I.getOperand(1));
4287 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
4288 TLI.getVectorIdxTy(DAG.getDataLayout()));
4290 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4291 InVec, InVal, InIdx));
4292}
4293
4294void SelectionDAGBuilder::visitBitInsert(const User &I) {
4295 SDValue Base = getValue(I.getOperand(0));
4296 SDValue Val = getValue(I.getOperand(1));
4297 SDValue Offset = getValue(I.getOperand(2));
4298 EVT BaseVT = Base.getValueType();
4299 EVT ValVT = Val.getValueType();
4300 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4301 SDLoc dl = getCurSDLoc();
4302
4303 assert(BaseVT.getScalarSizeInBits() >= ValVT.getScalarSizeInBits() &&
4304 "bitinsert val wider than base should be rejected by verifier");
4305
4306 // If Val is a float, cast it to an integer of the same bitwidth
4307 // so DAG.getZExtOrTrunc can process it safely.
4308 if (!ValVT.isInteger()) {
4309 ValVT = ValVT.changeTypeToInteger();
4310 Val = DAG.getBitcast(ValVT, Val);
4311 }
4312
4313 // Legalize shift amount to the target's shift amount type.
4314 EVT ShiftAmtTy = TLI.getShiftAmountTy(BaseVT, DAG.getDataLayout());
4315 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Offset, dl, ShiftAmtTy);
4316
4317 unsigned BaseBitWidth = BaseVT.getScalarSizeInBits();
4318 unsigned ValBitWidth = ValVT.getScalarSizeInBits();
4319 APInt InsertMask = APInt::getLowBitsSet(BaseBitWidth, ValBitWidth);
4320 SDValue ShiftedMask =
4321 DAG.getNode(ISD::SHL, dl, BaseVT, DAG.getConstant(InsertMask, dl, BaseVT),
4322 LegalShiftAmount);
4323 SDValue ClearMask = DAG.getNOT(dl, ShiftedMask, BaseVT);
4324 SDValue ClearedBase = DAG.getNode(ISD::AND, dl, BaseVT, Base, ClearMask);
4325
4326 SDValue ExtVal = DAG.getZExtOrTrunc(Val, dl, BaseVT);
4327 SDValue ShiftedVal =
4328 DAG.getNode(ISD::SHL, dl, BaseVT, ExtVal, LegalShiftAmount);
4329 SDValue Result = DAG.getNode(ISD::OR, dl, BaseVT, ClearedBase, ShiftedVal);
4330 setValue(&I, Result);
4331}
4332
4333void SelectionDAGBuilder::visitBitExtract(const User &I) {
4334 SDValue Src = getValue(I.getOperand(0));
4335 SDValue Offset = getValue(I.getOperand(1));
4336 EVT SrcVT = Src.getValueType();
4337 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4338 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4339 SDLoc dl = getCurSDLoc();
4340
4341 assert(ResultVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() &&
4342 "bitextract result wider than source should be rejected by verifier");
4343
4344 // Legalize shift amount to the target's shift amount type.
4345 EVT ShiftAmtTy = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
4346 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Offset, dl, ShiftAmtTy);
4347
4348 // Shift right by Offset - brings target field to bit 0
4349 SDValue Shifted = DAG.getNode(ISD::SRL, dl, SrcVT, Src, LegalShiftAmount);
4350
4352 if (!ResultVT.isInteger()) {
4353 // Drop into the integer domain to safely truncate the shifted bits
4354 EVT IntResultVT = ResultVT.changeTypeToInteger();
4355 Result = DAG.getNode(ISD::TRUNCATE, dl, IntResultVT, Shifted);
4356 Result = DAG.getBitcast(ResultVT, Result);
4357 } else {
4358 // Normal integer path
4359 Result = DAG.getNode(ISD::TRUNCATE, dl, ResultVT, Shifted);
4360 }
4361
4362 setValue(&I, Result);
4363}
4364
4365void SelectionDAGBuilder::visitExtractElement(const User &I) {
4366 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4367 SDValue InVec = getValue(I.getOperand(0));
4368 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
4369 TLI.getVectorIdxTy(DAG.getDataLayout()));
4371 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4372 InVec, InIdx));
4373}
4374
4375void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4376 SDValue Src1 = getValue(I.getOperand(0));
4377 SDValue Src2 = getValue(I.getOperand(1));
4378 ArrayRef<int> Mask;
4379 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
4380 Mask = SVI->getShuffleMask();
4381 else
4382 Mask = cast<ConstantExpr>(I).getShuffleMask();
4383 SDLoc DL = getCurSDLoc();
4384 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4385 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4386 EVT SrcVT = Src1.getValueType();
4387
4388 if (all_of(Mask, equal_to(0)) && VT.isScalableVector()) {
4389 // Canonical splat form of first element of first input vector.
4390 SDValue FirstElt =
4391 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
4392 DAG.getVectorIdxConstant(0, DL));
4393 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4394 return;
4395 }
4396
4397 // For now, we only handle splats for scalable vectors.
4398 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4399 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4400 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4401
4402 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4403 unsigned MaskNumElts = Mask.size();
4404
4405 if (SrcNumElts == MaskNumElts) {
4406 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4407 return;
4408 }
4409
4410 // Normalize the shuffle vector since mask and vector length don't match.
4411 if (SrcNumElts < MaskNumElts) {
4412 // Mask is longer than the source vectors. We can use concatenate vector to
4413 // make the mask and vectors lengths match.
4414
4415 if (MaskNumElts % SrcNumElts == 0) {
4416 // Mask length is a multiple of the source vector length.
4417 // Check if the shuffle is some kind of concatenation of the input
4418 // vectors.
4419 unsigned NumConcat = MaskNumElts / SrcNumElts;
4420 bool IsConcat = true;
4421 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4422 for (unsigned i = 0; i != MaskNumElts; ++i) {
4423 int Idx = Mask[i];
4424 if (Idx < 0)
4425 continue;
4426 // Ensure the indices in each SrcVT sized piece are sequential and that
4427 // the same source is used for the whole piece.
4428 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4429 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4430 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4431 IsConcat = false;
4432 break;
4433 }
4434 // Remember which source this index came from.
4435 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4436 }
4437
4438 // The shuffle is concatenating multiple vectors together. Just emit
4439 // a CONCAT_VECTORS operation.
4440 if (IsConcat) {
4441 SmallVector<SDValue, 8> ConcatOps;
4442 for (auto Src : ConcatSrcs) {
4443 if (Src < 0)
4444 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4445 else if (Src == 0)
4446 ConcatOps.push_back(Src1);
4447 else
4448 ConcatOps.push_back(Src2);
4449 }
4450 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4451 return;
4452 }
4453 }
4454
4455 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4456 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4457 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4458 PaddedMaskNumElts);
4459
4460 // Pad both vectors with undefs to make them the same length as the mask.
4461 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4462
4463 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4464 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4465 MOps1[0] = Src1;
4466 MOps2[0] = Src2;
4467
4468 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4469 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4470
4471 // Readjust mask for new input vector length.
4472 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4473 for (unsigned i = 0; i != MaskNumElts; ++i) {
4474 int Idx = Mask[i];
4475 if (Idx >= (int)SrcNumElts)
4476 Idx -= SrcNumElts - PaddedMaskNumElts;
4477 MappedOps[i] = Idx;
4478 }
4479
4480 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4481
4482 // If the concatenated vector was padded, extract a subvector with the
4483 // correct number of elements.
4484 if (MaskNumElts != PaddedMaskNumElts)
4485 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
4486 DAG.getVectorIdxConstant(0, DL));
4487
4488 setValue(&I, Result);
4489 return;
4490 }
4491
4492 assert(SrcNumElts > MaskNumElts);
4493
4494 // Analyze the access pattern of the vector to see if we can extract
4495 // two subvectors and do the shuffle.
4496 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4497 bool CanExtract = true;
4498 for (int Idx : Mask) {
4499 unsigned Input = 0;
4500 if (Idx < 0)
4501 continue;
4502
4503 if (Idx >= (int)SrcNumElts) {
4504 Input = 1;
4505 Idx -= SrcNumElts;
4506 }
4507
4508 // If all the indices come from the same MaskNumElts sized portion of
4509 // the sources we can use extract. Also make sure the extract wouldn't
4510 // extract past the end of the source.
4511 int NewStartIdx = alignDown(Idx, MaskNumElts);
4512 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4513 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4514 CanExtract = false;
4515 // Make sure we always update StartIdx as we use it to track if all
4516 // elements are undef.
4517 StartIdx[Input] = NewStartIdx;
4518 }
4519
4520 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4521 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4522 return;
4523 }
4524 if (CanExtract) {
4525 // Extract appropriate subvector and generate a vector shuffle
4526 for (unsigned Input = 0; Input < 2; ++Input) {
4527 SDValue &Src = Input == 0 ? Src1 : Src2;
4528 if (StartIdx[Input] < 0)
4529 Src = DAG.getUNDEF(VT);
4530 else {
4531 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4532 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4533 }
4534 }
4535
4536 // Calculate new mask.
4537 SmallVector<int, 8> MappedOps(Mask);
4538 for (int &Idx : MappedOps) {
4539 if (Idx >= (int)SrcNumElts)
4540 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4541 else if (Idx >= 0)
4542 Idx -= StartIdx[0];
4543 }
4544
4545 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4546 return;
4547 }
4548
4549 // We can't use either concat vectors or extract subvectors so fall back to
4550 // replacing the shuffle with extract and build vector.
4551 // to insert and build vector.
4552 EVT EltVT = VT.getVectorElementType();
4554 for (int Idx : Mask) {
4555 SDValue Res;
4556
4557 if (Idx < 0) {
4558 Res = DAG.getUNDEF(EltVT);
4559 } else {
4560 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4561 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4562
4563 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4564 DAG.getVectorIdxConstant(Idx, DL));
4565 }
4566
4567 Ops.push_back(Res);
4568 }
4569
4570 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4571}
4572
4573void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4574 ArrayRef<unsigned> Indices = I.getIndices();
4575 const Value *Op0 = I.getOperand(0);
4576 const Value *Op1 = I.getOperand(1);
4577 Type *AggTy = I.getType();
4578 Type *ValTy = Op1->getType();
4579 bool IntoUndef = isa<UndefValue>(Op0);
4580 bool FromUndef = isa<UndefValue>(Op1);
4581
4582 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4583
4584 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4585 SmallVector<EVT, 4> AggValueVTs;
4586 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4587 SmallVector<EVT, 4> ValValueVTs;
4588 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4589
4590 unsigned NumAggValues = AggValueVTs.size();
4591 unsigned NumValValues = ValValueVTs.size();
4592 SmallVector<SDValue, 4> Values(NumAggValues);
4593
4594 // Ignore an insertvalue that produces an empty object
4595 if (!NumAggValues) {
4596 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4597 return;
4598 }
4599
4600 SDValue Agg = getValue(Op0);
4601 unsigned i = 0;
4602 // Copy the beginning value(s) from the original aggregate.
4603 for (; i != LinearIndex; ++i)
4604 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4605 SDValue(Agg.getNode(), Agg.getResNo() + i);
4606 // Copy values from the inserted value(s).
4607 if (NumValValues) {
4608 SDValue Val = getValue(Op1);
4609 for (; i != LinearIndex + NumValValues; ++i)
4610 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4611 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4612 }
4613 // Copy remaining value(s) from the original aggregate.
4614 for (; i != NumAggValues; ++i)
4615 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4616 SDValue(Agg.getNode(), Agg.getResNo() + i);
4617
4619 DAG.getVTList(AggValueVTs), Values));
4620}
4621
4622void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4623 ArrayRef<unsigned> Indices = I.getIndices();
4624 const Value *Op0 = I.getOperand(0);
4625 Type *AggTy = Op0->getType();
4626 Type *ValTy = I.getType();
4627 bool OutOfUndef = isa<UndefValue>(Op0);
4628
4629 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4630
4631 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4632 SmallVector<EVT, 4> ValValueVTs;
4633 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4634
4635 unsigned NumValValues = ValValueVTs.size();
4636
4637 // Ignore a extractvalue that produces an empty object
4638 if (!NumValValues) {
4639 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4640 return;
4641 }
4642
4643 SmallVector<SDValue, 4> Values(NumValValues);
4644
4645 SDValue Agg = getValue(Op0);
4646 // Copy out the selected value(s).
4647 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4648 Values[i - LinearIndex] =
4649 OutOfUndef ?
4650 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4651 SDValue(Agg.getNode(), Agg.getResNo() + i);
4652
4654 DAG.getVTList(ValValueVTs), Values));
4655}
4656
4657void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4658 Value *Op0 = I.getOperand(0);
4659 // Note that the pointer operand may be a vector of pointers. Take the scalar
4660 // element which holds a pointer.
4661 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4662 SDValue N = getValue(Op0);
4663 SDLoc dl = getCurSDLoc();
4664 auto &TLI = DAG.getTargetLoweringInfo();
4665 GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();
4666
4667 // For a vector GEP, keep the prefix scalar as long as possible, then
4668 // convert any scalars encountered after the first vector operand to vectors.
4669 bool IsVectorGEP = I.getType()->isVectorTy();
4670 ElementCount VectorElementCount =
4671 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4673
4675 GTI != E; ++GTI) {
4676 const Value *Idx = GTI.getOperand();
4677 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4678 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4679 if (Field) {
4680 // N = N + Offset
4682 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
4683
4684 // In an inbounds GEP with an offset that is nonnegative even when
4685 // interpreted as signed, assume there is no unsigned overflow.
4686 SDNodeFlags Flags;
4687 if (NW.hasNoUnsignedWrap() ||
4688 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4690 Flags.setInBounds(NW.isInBounds());
4691
4692 N = DAG.getMemBasePlusOffset(
4693 N, DAG.getConstant(Offset, dl, N.getValueType()), dl, Flags);
4694 }
4695 } else {
4696 // IdxSize is the width of the arithmetic according to IR semantics.
4697 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4698 // (and fix up the result later).
4699 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4700 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4701 TypeSize ElementSize =
4702 GTI.getSequentialElementStride(DAG.getDataLayout());
4703 // We intentionally mask away the high bits here; ElementSize may not
4704 // fit in IdxTy.
4705 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4706 /*isSigned=*/false, /*implicitTrunc=*/true);
4707 bool ElementScalable = ElementSize.isScalable();
4708
4709 // If this is a scalar constant or a splat vector of constants,
4710 // handle it quickly.
4711 const auto *C = dyn_cast<Constant>(Idx);
4712 if (C && isa<VectorType>(C->getType()))
4713 C = C->getSplatValue();
4714
4715 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4716 if (CI && CI->isZero())
4717 continue;
4718 if (CI && !ElementScalable) {
4719 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4720 LLVMContext &Context = *DAG.getContext();
4721 SDValue OffsVal;
4722 if (N.getValueType().isVector())
4723 OffsVal = DAG.getConstant(
4724 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4725 else
4726 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4727
4728 // In an inbounds GEP with an offset that is nonnegative even when
4729 // interpreted as signed, assume there is no unsigned overflow.
4730 SDNodeFlags Flags;
4731 if (NW.hasNoUnsignedWrap() ||
4732 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4733 Flags.setNoUnsignedWrap(true);
4734 Flags.setInBounds(NW.isInBounds());
4735
4736 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4737
4738 N = DAG.getMemBasePlusOffset(N, OffsVal, dl, Flags);
4739 continue;
4740 }
4741
4742 // N = N + Idx * ElementMul;
4743 SDValue IdxN = getValue(Idx);
4744
4745 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
4746 if (N.getValueType().isVector()) {
4747 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
4748 VectorElementCount);
4749 IdxN = DAG.getSplat(VT, dl, IdxN);
4750 } else {
4751 EVT VT =
4752 EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
4753 N = DAG.getSplat(VT, dl, N);
4754 }
4755 }
4756
4757 // If the index is smaller or larger than intptr_t, truncate or extend
4758 // it.
4759 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4760
4761 SDNodeFlags ScaleFlags;
4762 // The multiplication of an index by the type size does not wrap the
4763 // pointer index type in a signed sense (mul nsw).
4765
4766 // The multiplication of an index by the type size does not wrap the
4767 // pointer index type in an unsigned sense (mul nuw).
4768 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4769
4770 if (ElementScalable) {
4771 EVT VScaleTy = N.getValueType().getScalarType();
4772 SDValue VScale = DAG.getNode(
4773 ISD::VSCALE, dl, VScaleTy,
4774 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4775 if (N.getValueType().isVector())
4776 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4777 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale,
4778 ScaleFlags);
4779 } else {
4780 // If this is a multiply by a power of two, turn it into a shl
4781 // immediately. This is a very common case.
4782 if (ElementMul != 1) {
4783 if (ElementMul.isPowerOf2()) {
4784 unsigned Amt = ElementMul.logBase2();
4785 IdxN = DAG.getNode(
4786 ISD::SHL, dl, N.getValueType(), IdxN,
4787 DAG.getShiftAmountConstant(Amt, N.getValueType(), dl),
4788 ScaleFlags);
4789 } else {
4790 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4791 IdxN.getValueType());
4792 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, Scale,
4793 ScaleFlags);
4794 }
4795 }
4796 }
4797
4798 // The successive addition of the current address, truncated to the
4799 // pointer index type and interpreted as an unsigned number, and each
4800 // offset, also interpreted as an unsigned number, does not wrap the
4801 // pointer index type (add nuw).
4802 SDNodeFlags AddFlags;
4803 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4804 AddFlags.setInBounds(NW.isInBounds());
4805
4806 N = DAG.getMemBasePlusOffset(N, IdxN, dl, AddFlags);
4807 }
4808 }
4809
4810 if (IsVectorGEP && !N.getValueType().isVector()) {
4811 EVT VT = EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
4812 N = DAG.getSplat(VT, dl, N);
4813 }
4814
4815 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4816 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4817 if (IsVectorGEP) {
4818 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4819 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4820 }
4821
4822 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4823 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4824
4825 setValue(&I, N);
4826}
4827
4828void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4829 // If this is a fixed sized alloca in the entry block of the function,
4830 // allocate it statically on the stack.
4831 if (FuncInfo.StaticAllocaMap.count(&I))
4832 return; // getValue will auto-populate this.
4833
4834 SDLoc dl = getCurSDLoc();
4835 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4836 auto &DL = DAG.getDataLayout();
4837 TypeSize TySize = I.getAllocationBaseSize(DL);
4838 MaybeAlign Alignment = I.getAlign();
4839
4840 SDValue AllocSize = getValue(I.getArraySize());
4841
4842 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4843 if (AllocSize.getValueType() != IntPtr)
4844 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4845
4846 AllocSize = DAG.getNode(
4847 ISD::MUL, dl, IntPtr, AllocSize,
4848 DAG.getZExtOrTrunc(DAG.getTypeSize(dl, MVT::i64, TySize), dl, IntPtr));
4849
4850 // Handle alignment. If the requested alignment is less than or equal to
4851 // the stack alignment, ignore it. If the size is greater than or equal to
4852 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4853 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4854 if (*Alignment <= StackAlign)
4855 Alignment = std::nullopt;
4856
4857 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4858 // Round the size of the allocation up to the stack alignment size
4859 // by add SA-1 to the size. This doesn't overflow because we're computing
4860 // an address inside an alloca.
4861 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4862 DAG.getConstant(StackAlignMask, dl, IntPtr),
4864
4865 // Mask out the low bits for alignment purposes.
4866 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4867 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
4868
4869 SDValue Ops[] = {
4870 getRoot(), AllocSize,
4871 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4872 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4873 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4874 setValue(&I, DSA);
4875 DAG.setRoot(DSA.getValue(1));
4876
4877 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4878}
4879
4880static const MDNode *getRangeMetadata(const Instruction &I) {
4881 return I.getMetadata(LLVMContext::MD_range);
4882}
4883
4884static std::optional<ConstantRange> getRange(const Instruction &I) {
4885 if (const auto *CB = dyn_cast<CallBase>(&I))
4886 if (std::optional<ConstantRange> CR = CB->getRange())
4887 return CR;
4888 if (const MDNode *Range = getRangeMetadata(I))
4890 return std::nullopt;
4891}
4892
4894 if (const auto *CB = dyn_cast<CallBase>(&I))
4895 return CB->getRetNoFPClass();
4896 return fcNone;
4897}
4898
4899void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4900 if (I.isAtomic())
4901 return visitAtomicLoad(I);
4902
4903 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4904 const Value *SV = I.getOperand(0);
4905 if (TLI.supportSwiftError()) {
4906 // Swifterror values can come from either a function parameter with
4907 // swifterror attribute or an alloca with swifterror attribute.
4908 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4909 if (Arg->hasSwiftErrorAttr())
4910 return visitLoadFromSwiftError(I);
4911 }
4912
4913 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4914 if (Alloca->isSwiftError())
4915 return visitLoadFromSwiftError(I);
4916 }
4917 }
4918
4919 SDValue Ptr = getValue(SV);
4920
4921 Type *Ty = I.getType();
4922 SmallVector<EVT, 4> ValueVTs, MemVTs;
4924 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4925 unsigned NumValues = ValueVTs.size();
4926 if (NumValues == 0)
4927 return;
4928
4929 Align Alignment = I.getAlign();
4930 AAMDNodes AAInfo = I.getAAMetadata();
4931 const MDNode *Ranges = getRangeMetadata(I);
4932 const MDNode *MemCacheHint = getMemCacheHint(I);
4933 bool isVolatile = I.isVolatile();
4934 MachineMemOperand::Flags MMOFlags =
4935 TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
4936
4937 SDValue Root;
4938 bool ConstantMemory = false;
4939 if (isVolatile)
4940 // Serialize volatile loads with other side effects.
4941 Root = getRoot();
4942 else if (NumValues > MaxParallelChains)
4943 Root = getMemoryRoot();
4944 else if (BatchAA &&
4945 BatchAA->pointsToConstantMemory(MemoryLocation(
4946 SV,
4947 LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4948 AAInfo))) {
4949 // Do not serialize (non-volatile) loads of constant memory with anything.
4950 Root = DAG.getEntryNode();
4951 ConstantMemory = true;
4953 } else {
4954 // Do not serialize non-volatile loads against each other.
4955 Root = DAG.getRoot();
4956 }
4957
4958 SDLoc dl = getCurSDLoc();
4959
4960 if (isVolatile)
4961 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4962
4964 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4965
4966 unsigned ChainI = 0;
4967 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4968 // Serializing loads here may result in excessive register pressure, and
4969 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4970 // could recover a bit by hoisting nodes upward in the chain by recognizing
4971 // they are side-effect free or do not alias. The optimizer should really
4972 // avoid this case by converting large object/array copies to llvm.memcpy
4973 // (MaxParallelChains should always remain as failsafe).
4974 if (ChainI == MaxParallelChains) {
4975 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4976 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4977 ArrayRef(Chains.data(), ChainI));
4978 Root = Chain;
4979 ChainI = 0;
4980 }
4981
4982 // TODO: MachinePointerInfo only supports a fixed length offset.
4983 MachinePointerInfo PtrInfo =
4984 !Offsets[i].isScalable() || Offsets[i].isZero()
4985 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4986 : MachinePointerInfo();
4987
4988 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4989 SDValue L =
4990 DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment, MMOFlags,
4991 MMOMetadata(AAInfo, Ranges, MemCacheHint));
4992 Chains[ChainI] = L.getValue(1);
4993
4994 if (MemVTs[i] != ValueVTs[i])
4995 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4996
4997 if (MDNode *NoFPClassMD = I.getMetadata(LLVMContext::MD_nofpclass)) {
4998 uint64_t FPTestInt =
4999 cast<ConstantInt>(
5000 cast<ConstantAsMetadata>(NoFPClassMD->getOperand(0))->getValue())
5001 ->getZExtValue();
5002 if (FPTestInt != fcNone) {
5003 SDValue FPTestConst =
5004 DAG.getTargetConstant(FPTestInt, SDLoc(), MVT::i32);
5005 L = DAG.getNode(ISD::AssertNoFPClass, dl, L.getValueType(), L,
5006 FPTestConst);
5007 }
5008 }
5009 Values[i] = L;
5010 }
5011
5012 if (!ConstantMemory) {
5013 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5014 ArrayRef(Chains.data(), ChainI));
5015 if (isVolatile)
5016 DAG.setRoot(Chain);
5017 else
5018 PendingLoads.push_back(Chain);
5019 }
5020
5021 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
5022 DAG.getVTList(ValueVTs), Values));
5023}
5024
5025void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
5026 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5027 "call visitStoreToSwiftError when backend supports swifterror");
5028
5029 SmallVector<EVT, 4> ValueVTs;
5030 SmallVector<uint64_t, 4> Offsets;
5031 const Value *SrcV = I.getOperand(0);
5032 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
5033 SrcV->getType(), ValueVTs, /*MemVTs=*/nullptr, &Offsets, 0);
5034 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5035 "expect a single EVT for swifterror");
5036
5037 SDValue Src = getValue(SrcV);
5038 // Create a virtual register, then update the virtual register.
5039 Register VReg =
5040 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
5041 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
5042 // Chain can be getRoot or getControlRoot.
5043 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
5044 SDValue(Src.getNode(), Src.getResNo()));
5045 DAG.setRoot(CopyNode);
5046}
5047
5048void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
5049 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5050 "call visitLoadFromSwiftError when backend supports swifterror");
5051
5052 assert(!I.isVolatile() &&
5053 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
5054 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
5055 "Support volatile, non temporal, invariant for load_from_swift_error");
5056
5057 const Value *SV = I.getOperand(0);
5058 Type *Ty = I.getType();
5059 assert(
5060 (!BatchAA ||
5061 !BatchAA->pointsToConstantMemory(MemoryLocation(
5062 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
5063 I.getAAMetadata()))) &&
5064 "load_from_swift_error should not be constant memory");
5065
5066 SmallVector<EVT, 4> ValueVTs;
5067 SmallVector<uint64_t, 4> Offsets;
5068 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
5069 ValueVTs, /*MemVTs=*/nullptr, &Offsets, 0);
5070 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5071 "expect a single EVT for swifterror");
5072
5073 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
5074 SDValue L = DAG.getCopyFromReg(
5075 getRoot(), getCurSDLoc(),
5076 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
5077
5078 setValue(&I, L);
5079}
5080
5081void SelectionDAGBuilder::visitStore(const StoreInst &I) {
5082 if (I.isAtomic())
5083 return visitAtomicStore(I);
5084
5085 const Value *SrcV = I.getOperand(0);
5086 const Value *PtrV = I.getOperand(1);
5087
5088 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5089 if (TLI.supportSwiftError()) {
5090 // Swifterror values can come from either a function parameter with
5091 // swifterror attribute or an alloca with swifterror attribute.
5092 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
5093 if (Arg->hasSwiftErrorAttr())
5094 return visitStoreToSwiftError(I);
5095 }
5096
5097 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
5098 if (Alloca->isSwiftError())
5099 return visitStoreToSwiftError(I);
5100 }
5101 }
5102
5103 SmallVector<EVT, 4> ValueVTs, MemVTs;
5105 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
5106 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
5107 unsigned NumValues = ValueVTs.size();
5108 if (NumValues == 0)
5109 return;
5110
5111 // Get the lowered operands. Note that we do this after
5112 // checking if NumResults is zero, because with zero results
5113 // the operands won't have values in the map.
5114 SDValue Src = getValue(SrcV);
5115 SDValue Ptr = getValue(PtrV);
5116
5117 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
5118 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
5119 SDLoc dl = getCurSDLoc();
5120 Align Alignment = I.getAlign();
5121 AAMDNodes AAInfo = I.getAAMetadata();
5122 const MDNode *MemCacheHint = getMemCacheHint(I, I.getPointerOperandIndex());
5123
5124 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
5125
5126 unsigned ChainI = 0;
5127 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
5128 // See visitLoad comments.
5129 if (ChainI == MaxParallelChains) {
5130 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5131 ArrayRef(Chains.data(), ChainI));
5132 Root = Chain;
5133 ChainI = 0;
5134 }
5135
5136 // TODO: MachinePointerInfo only supports a fixed length offset.
5137 MachinePointerInfo PtrInfo =
5138 !Offsets[i].isScalable() || Offsets[i].isZero()
5139 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
5140 : MachinePointerInfo();
5141
5142 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
5143 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
5144 if (MemVTs[i] != ValueVTs[i])
5145 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
5146 SDValue St =
5147 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags,
5148 MMOMetadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
5149 Chains[ChainI] = St;
5150 }
5151
5152 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5153 ArrayRef(Chains.data(), ChainI));
5154 setValue(&I, StoreNode);
5155 DAG.setRoot(StoreNode);
5156}
5157
5158void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
5159 bool IsCompressing) {
5160 SDLoc sdl = getCurSDLoc();
5161
5162 Value *Src0Operand = I.getArgOperand(0);
5163 Value *PtrOperand = I.getArgOperand(1);
5164 Value *MaskOperand = I.getArgOperand(2);
5165 Align Alignment = I.getParamAlign(1).valueOrOne();
5166
5167 SDValue Ptr = getValue(PtrOperand);
5168 SDValue Src0 = getValue(Src0Operand);
5169 SDValue Mask = getValue(MaskOperand);
5170 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
5171
5172 EVT VT = Src0.getValueType();
5173
5174 const auto &TLI = DAG.getTargetLoweringInfo();
5175
5176 auto MMOFlags = MachineMemOperand::MOStore;
5177 MMOFlags |= TLI.getTargetMMOFlags(I);
5178 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5180
5181 const MDNode *MemCacheHint = getMemCacheHint(I, /*OperandNo=*/1);
5182
5183 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5184 MachinePointerInfo(PtrOperand), MMOFlags,
5185 LocationSize::upperBound(VT.getStoreSize()), Alignment,
5186 MMOMetadata(I.getAAMetadata(), /*Ranges=*/nullptr, MemCacheHint));
5187
5188 SDValue StoreNode =
5189 !IsCompressing && TTI->hasConditionalLoadStoreForType(
5190 I.getArgOperand(0)->getType(), /*IsStore=*/true)
5191 ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0,
5192 Mask)
5193 : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask,
5194 VT, MMO, ISD::UNINDEXED, /*Truncating=*/false,
5195 IsCompressing);
5196 DAG.setRoot(StoreNode);
5197 setValue(&I, StoreNode);
5198}
5199
5200// Get a uniform base for the Gather/Scatter intrinsic.
5201// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
5202// We try to represent it as a base pointer + vector of indices.
5203// Usually, the vector of pointers comes from a 'getelementptr' instruction.
5204// The first operand of the GEP may be a single pointer or a vector of pointers
5205// Example:
5206// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
5207// or
5208// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
5209// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
5210//
5211// When the first GEP operand is a single pointer - it is the uniform base we
5212// are looking for. If first operand of the GEP is a splat vector - we
5213// extract the splat value and use it as a uniform base.
5214// In all other cases the function returns 'false'.
5215static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
5216 SDValue &Scale, SelectionDAGBuilder *SDB,
5217 const BasicBlock *CurBB, uint64_t ElemSize) {
5218 SelectionDAG& DAG = SDB->DAG;
5220 const DataLayout &DL = DAG.getDataLayout();
5221
5222 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
5223
5224 // Handle splat constant pointer.
5225 if (auto *C = dyn_cast<Constant>(Ptr)) {
5226 C = C->getSplatValue();
5227 if (!C)
5228 return false;
5229
5230 Base = SDB->getValue(C);
5231
5232 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
5233 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
5234 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
5235 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
5236 return true;
5237 }
5238
5240 if (!GEP || GEP->getParent() != CurBB)
5241 return false;
5242
5243 if (GEP->getNumOperands() != 2)
5244 return false;
5245
5246 const Value *BasePtr = GEP->getPointerOperand();
5247 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
5248
5249 // Make sure the base is scalar and the index is a vector.
5250 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
5251 return false;
5252
5253 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
5254 if (ScaleVal.isScalable())
5255 return false;
5256
5257 // Target may not support the required addressing mode.
5258 if (ScaleVal != 1 &&
5259 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
5260 return false;
5261
5262 Base = SDB->getValue(BasePtr);
5263 Index = SDB->getValue(IndexVal);
5264
5265 Scale =
5266 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
5267 return true;
5268}
5269
5270void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
5271 SDLoc sdl = getCurSDLoc();
5272
5273 // llvm.masked.scatter.*(Src0, Ptrs, Mask)
5274 const Value *Ptr = I.getArgOperand(1);
5275 SDValue Src0 = getValue(I.getArgOperand(0));
5276 SDValue Mask = getValue(I.getArgOperand(2));
5277 EVT VT = Src0.getValueType();
5278 Align Alignment = I.getParamAlign(1).valueOrOne();
5279 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5280
5281 SDValue Base;
5282 SDValue Index;
5283 SDValue Scale;
5284 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
5285 I.getParent(), VT.getScalarStoreSize());
5286
5287 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5288 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5289 MachinePointerInfo(AS), MachineMemOperand::MOStore,
5290 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
5291 if (!UniformBase) {
5292 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5293 Index = getValue(Ptr);
5294 Scale =
5295 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5296 }
5297
5298 EVT IdxVT = Index.getValueType();
5299 EVT EltTy = IdxVT.getVectorElementType();
5300 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5301 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
5302 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5303 }
5304
5305 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
5306 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
5307 Ops, MMO, ISD::SIGNED_SCALED, false);
5308 DAG.setRoot(Scatter);
5309 setValue(&I, Scatter);
5310}
5311
5312void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
5313 SDLoc sdl = getCurSDLoc();
5314
5315 Value *PtrOperand = I.getArgOperand(0);
5316 Value *MaskOperand = I.getArgOperand(1);
5317 Value *Src0Operand = I.getArgOperand(2);
5318 Align Alignment = I.getParamAlign(0).valueOrOne();
5319
5320 SDValue Ptr = getValue(PtrOperand);
5321 SDValue Src0 = getValue(Src0Operand);
5322 SDValue Mask = getValue(MaskOperand);
5323 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
5324
5325 EVT VT = Src0.getValueType();
5326 AAMDNodes AAInfo = I.getAAMetadata();
5327 const MDNode *Ranges = getRangeMetadata(I);
5328 const MDNode *MemCacheHint = getMemCacheHint(I, /*OperandNo=*/0);
5329
5330 // Do not serialize masked loads of constant memory with anything.
5331 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
5332 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
5333
5334 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5335
5336 const auto &TLI = DAG.getTargetLoweringInfo();
5337
5338 auto MMOFlags = MachineMemOperand::MOLoad;
5339 MMOFlags |= TLI.getTargetMMOFlags(I);
5340 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5342 if (I.hasMetadata(LLVMContext::MD_invariant_load))
5344
5345 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5346 MachinePointerInfo(PtrOperand), MMOFlags,
5347 LocationSize::upperBound(VT.getStoreSize()), Alignment,
5348 MMOMetadata(AAInfo, Ranges, MemCacheHint));
5349
5350 // The Load/Res may point to different values and both of them are output
5351 // variables.
5352 SDValue Load;
5353 SDValue Res;
5354 if (!IsExpanding &&
5355 TTI->hasConditionalLoadStoreForType(Src0Operand->getType(),
5356 /*IsStore=*/false))
5357 Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask);
5358 else
5359 Res = Load =
5360 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
5361 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5362 if (AddToChain)
5363 PendingLoads.push_back(Load.getValue(1));
5364 setValue(&I, Res);
5365}
5366
5367void SelectionDAGBuilder::visitSpeculativeLoad(const CallInst &I) {
5368 SDLoc sdl = getCurSDLoc();
5369 Value *PtrOperand = I.getArgOperand(0);
5370 // The remaining arguments (num_accessible_bytes or oracle function + args)
5371 // are IR-level semantics only; they are not needed at codegen.
5372 SDValue Ptr = getValue(PtrOperand);
5373
5374 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5375 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5376 Align Alignment = I.getParamAlign(0).valueOrOne();
5377 AAMDNodes AAInfo = I.getAAMetadata();
5378
5379 SDValue InChain = DAG.getRoot();
5380
5381 // Use MOLoad but NOT MODereferenceable - the memory may not be
5382 // fully dereferenceable.
5383 auto MMOFlags = MachineMemOperand::MOLoad;
5384 MMOFlags |= TLI.getTargetMMOFlags(I);
5385 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5387 if (I.hasMetadata(LLVMContext::MD_invariant_load))
5389
5390 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5391 MachinePointerInfo(PtrOperand), MMOFlags,
5392 LocationSize::precise(VT.getStoreSize()), Alignment, AAInfo);
5393
5394 SDValue Load = DAG.getLoad(VT, sdl, InChain, Ptr, MMO);
5395 PendingLoads.push_back(Load.getValue(1));
5396 setValue(&I, Load);
5397}
5398
5399void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5400 SDLoc sdl = getCurSDLoc();
5401
5402 // @llvm.masked.gather.*(Ptrs, Mask, Src0)
5403 const Value *Ptr = I.getArgOperand(0);
5404 SDValue Src0 = getValue(I.getArgOperand(2));
5405 SDValue Mask = getValue(I.getArgOperand(1));
5406
5407 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5408 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5409 Align Alignment = I.getParamAlign(0).valueOrOne();
5410
5411 const MDNode *Ranges = getRangeMetadata(I);
5412
5413 SDValue Root = DAG.getRoot();
5414 SDValue Base;
5415 SDValue Index;
5416 SDValue Scale;
5417 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
5418 I.getParent(), VT.getScalarStoreSize());
5419 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5420 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5421 MachinePointerInfo(AS), MachineMemOperand::MOLoad,
5423 MMOMetadata(I.getAAMetadata(), Ranges));
5424
5425 if (!UniformBase) {
5426 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5427 Index = getValue(Ptr);
5428 Scale =
5429 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5430 }
5431
5432 EVT IdxVT = Index.getValueType();
5433 EVT EltTy = IdxVT.getVectorElementType();
5434 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5435 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
5436 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5437 }
5438
5439 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5440 SDValue Gather =
5441 DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, Ops, MMO,
5443
5444 PendingLoads.push_back(Gather.getValue(1));
5445 setValue(&I, Gather);
5446}
5447
5448void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5449 SDLoc dl = getCurSDLoc();
5450 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5451 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5452 SyncScope::ID SSID = I.getSyncScopeID();
5453
5454 SDValue InChain = getRoot();
5455
5456 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
5457 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5458
5459 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5460 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
5461
5462 MachineFunction &MF = DAG.getMachineFunction();
5463 const MDNode *MemCacheHint = getMemCacheHint(I);
5464 MachineMemOperand *MMO = MF.getMachineMemOperand(
5465 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5466 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5467 SSID, SuccessOrdering, FailureOrdering);
5468
5470 dl, MemVT, VTs, InChain,
5471 getValue(I.getPointerOperand()),
5472 getValue(I.getCompareOperand()),
5473 getValue(I.getNewValOperand()), MMO);
5474
5475 SDValue OutChain = L.getValue(2);
5476
5477 setValue(&I, L);
5478 DAG.setRoot(OutChain);
5479}
5480
5481void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5482 SDLoc dl = getCurSDLoc();
5484 switch (I.getOperation()) {
5485 default: llvm_unreachable("Unknown atomicrmw operation");
5503 break;
5506 break;
5509 break;
5512 break;
5515 break;
5518 break;
5521 break;
5524 break;
5525 }
5526 AtomicOrdering Ordering = I.getOrdering();
5527 SyncScope::ID SSID = I.getSyncScopeID();
5528
5529 SDValue InChain = getRoot();
5530
5531 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5532 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5533 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
5534
5535 MachineFunction &MF = DAG.getMachineFunction();
5536 const MDNode *MemCacheHint = getMemCacheHint(I);
5537 MachineMemOperand *MMO = MF.getMachineMemOperand(
5538 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5539 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5540 SSID, Ordering);
5541
5542 SDValue L =
5543 DAG.getAtomic(NT, dl, MemVT, InChain,
5544 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5545 MMO);
5546
5547 SDValue OutChain = L.getValue(1);
5548
5549 setValue(&I, L);
5550 DAG.setRoot(OutChain);
5551}
5552
5553void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5554 SDLoc dl = getCurSDLoc();
5555 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5556 SDValue Ops[3];
5557 Ops[0] = getRoot();
5558 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5559 TLI.getFenceOperandTy(DAG.getDataLayout()));
5560 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5561 TLI.getFenceOperandTy(DAG.getDataLayout()));
5562 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5563 setValue(&I, N);
5564 DAG.setRoot(N);
5565}
5566
5567void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5568 SDLoc dl = getCurSDLoc();
5569 AtomicOrdering Order = I.getOrdering();
5570 SyncScope::ID SSID = I.getSyncScopeID();
5571
5572 SDValue InChain = getRoot();
5573
5574 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5575 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5576 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5577
5578 if (!TLI.isAtomicAlignmentSupported(I.getAlign(), MemVT.getSizeInBits() / 8))
5579 report_fatal_error("Cannot generate unaligned atomic load");
5580
5581 auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
5582
5583 const MDNode *Ranges = getRangeMetadata(I);
5584 const MDNode *MemCacheHint = getMemCacheHint(I);
5585 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5586 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5587 I.getAlign(), MMOMetadata(AAMDNodes(), Ranges, MemCacheHint), SSID,
5588 Order);
5589
5590 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5591
5592 SDValue Ptr = getValue(I.getPointerOperand());
5593 SDValue L =
5594 DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, MemVT, MemVT, InChain, Ptr, MMO);
5595
5596 SDValue OutChain = L.getValue(1);
5597 if (MemVT != VT)
5598 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5599
5600 setValue(&I, L);
5601 DAG.setRoot(OutChain);
5602}
5603
5604void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5605 SDLoc dl = getCurSDLoc();
5606
5607 AtomicOrdering Ordering = I.getOrdering();
5608 SyncScope::ID SSID = I.getSyncScopeID();
5609
5610 SDValue InChain = getRoot();
5611
5612 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5613 EVT MemVT =
5614 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5615
5616 if (!TLI.isAtomicAlignmentSupported(I.getAlign(), MemVT.getSizeInBits() / 8))
5617 report_fatal_error("Cannot generate unaligned atomic store");
5618
5619 auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
5620
5621 MachineFunction &MF = DAG.getMachineFunction();
5622 const MDNode *MemCacheHint = getMemCacheHint(I, I.getPointerOperandIndex());
5623 MachineMemOperand *MMO = MF.getMachineMemOperand(
5624 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5625 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5626 SSID, Ordering);
5627
5628 SDValue Val = getValue(I.getValueOperand());
5629 if (Val.getValueType() != MemVT)
5630 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5631 SDValue Ptr = getValue(I.getPointerOperand());
5632
5633 SDValue OutChain =
5634 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5635
5636 setValue(&I, OutChain);
5637 DAG.setRoot(OutChain);
5638}
5639
5640/// Check if this intrinsic call depends on the chain (1st return value)
5641/// and if it only *loads* memory.
5642/// Ignore the callsite's attributes. A specific call site may be marked with
5643/// readnone, but the lowering code will expect the chain based on the
5644/// definition.
5645std::pair<bool, bool>
5646SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {
5647 const Function *F = I.getCalledFunction();
5648 bool HasChain = !F->doesNotAccessMemory();
5649 bool OnlyLoad =
5650 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5651
5652 return {HasChain, OnlyLoad};
5653}
5654
5655SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
5656 const CallBase &I, bool HasChain, bool OnlyLoad,
5657 TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
5658 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5659
5660 // Build the operand list.
5662 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5663 if (OnlyLoad) {
5664 // We don't need to serialize loads against other loads.
5665 Ops.push_back(DAG.getRoot());
5666 } else {
5667 Ops.push_back(getRoot());
5668 }
5669 }
5670
5671 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5672 if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||
5673 TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)
5674 Ops.push_back(DAG.getTargetConstant(I.getIntrinsicID(), getCurSDLoc(),
5675 TLI.getPointerTy(DAG.getDataLayout())));
5676
5677 // Add all operands of the call to the operand list.
5678 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5679 const Value *Arg = I.getArgOperand(i);
5680 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5681 Ops.push_back(getValue(Arg));
5682 continue;
5683 }
5684
5685 // Use TargetConstant instead of a regular constant for immarg.
5686 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5687 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5688 assert(CI->getBitWidth() <= 64 &&
5689 "large intrinsic immediates not handled");
5690 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5691 } else {
5692 Ops.push_back(
5693 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5694 }
5695 }
5696
5697 if (std::optional<OperandBundleUse> Bundle =
5698 I.getOperandBundle(LLVMContext::OB_deactivation_symbol)) {
5699 auto *Sym = Bundle->Inputs[0].get();
5700 SDValue SDSym = getValue(Sym);
5701 SDSym = DAG.getDeactivationSymbol(cast<GlobalValue>(Sym));
5702 Ops.push_back(SDSym);
5703 }
5704
5705 if (std::optional<OperandBundleUse> Bundle =
5706 I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5707 Value *Token = Bundle->Inputs[0].get();
5708 SDValue ConvControlToken = getValue(Token);
5709 assert(Ops.back().getValueType() != MVT::Glue &&
5710 "Did not expect another glue node here.");
5711 ConvControlToken =
5712 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5713 Ops.push_back(ConvControlToken);
5714 }
5715
5716 return Ops;
5717}
5718
5719SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
5720 bool HasChain) {
5721 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5722
5723 SmallVector<EVT, 4> ValueVTs;
5724 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5725
5726 if (HasChain)
5727 ValueVTs.push_back(MVT::Other);
5728
5729 return DAG.getVTList(ValueVTs);
5730}
5731
5732/// Get an INTRINSIC node for a target intrinsic which does not touch memory.
5733SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5734 const Type &IntrinsicVT, bool HasChain, ArrayRef<SDValue> Ops,
5735 const SDVTList &VTs) {
5736 if (!HasChain)
5737 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
5738 if (!IntrinsicVT.isVoidTy())
5739 return DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
5740 return DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
5741}
5742
5743/// Set root, convert return type if necessary and check alignment.
5744SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
5745 bool HasChain,
5746 bool OnlyLoad,
5747 SDValue Result) {
5748 if (HasChain) {
5749 SDValue Chain = Result.getValue(Result.getNode()->getNumValues() - 1);
5750 if (OnlyLoad)
5751 PendingLoads.push_back(Chain);
5752 else
5753 DAG.setRoot(Chain);
5754 }
5755
5756 if (I.getType()->isVoidTy())
5757 return Result;
5758
5759 if (MaybeAlign Alignment = I.getRetAlign(); InsertAssertAlign && Alignment) {
5760 // Insert `assertalign` node if there's an alignment.
5761 Result = DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5762 } else if (!isa<VectorType>(I.getType())) {
5763 Result = lowerRangeToAssertZExt(DAG, I, Result);
5764 }
5765
5766 return Result;
5767}
5768
5769/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5770/// node.
5771void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5772 unsigned Intrinsic) {
5773 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
5774 Intrinsic::ID IntrinsicID = static_cast<Intrinsic::ID>(Intrinsic);
5775
5776 if (!DAG.getMachineFunction().getSubtarget().isIntrinsicSupported(
5777 Intrinsic)) {
5778 SDLoc DL = getCurSDLoc();
5779 DAG.getContext()->diagnose(DiagnosticInfoUnsupportedTargetIntrinsic(
5780 *I.getFunction(), IntrinsicID, DL.getDebugLoc()));
5781
5782 // The intrinsic is not available on this subtarget. Preserve the chain for
5783 // side-effecting intrinsics and lower any result to poison so that
5784 // compilation can continue and collect further diagnostics.
5785 if (HasChain && !OnlyLoad)
5786 DAG.setRoot(getRoot());
5787
5789 return;
5790 }
5791
5792 // Infos is set by getTgtMemIntrinsic.
5794 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5795 TLI.getTgtMemIntrinsic(Infos, I, DAG.getMachineFunction(), Intrinsic);
5796 // Use the first (primary) info determines the node opcode.
5797 TargetLowering::IntrinsicInfo *Info = !Infos.empty() ? &Infos[0] : nullptr;
5798
5800 getTargetIntrinsicOperands(I, HasChain, OnlyLoad, Info);
5801 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
5802
5803 // Propagate fast-math-flags from IR to node(s).
5804 SDNodeFlags Flags;
5805 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5806 Flags.copyFMF(*FPMO);
5807 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5808
5809 // Create the node.
5811
5812 // In some cases, custom collection of operands from CallInst I may be needed.
5814 if (!Infos.empty()) {
5815 // This is target intrinsic that touches memory
5816 // Create MachineMemOperands for each memory access described by the target.
5817 MachineFunction &MF = DAG.getMachineFunction();
5819 for (const auto &Info : Infos) {
5820 // TODO: We currently just fallback to address space 0 if
5821 // getTgtMemIntrinsic didn't yield anything useful.
5822 MachinePointerInfo MPI;
5823 if (Info.ptrVal)
5824 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5825 else if (Info.fallbackAddressSpace)
5826 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5827 EVT MemVT = Info.memVT;
5828 LocationSize Size = LocationSize::precise(Info.size);
5829 if (Size.hasValue() && !Size.getValue())
5831 Align Alignment = Info.align.value_or(DAG.getEVTAlign(MemVT));
5832 MachineMemOperand *MMO = MF.getMachineMemOperand(
5833 MPI, Info.flags, Size, Alignment, I.getAAMetadata(), Info.ssid,
5834 Info.order, Info.failureOrder);
5835 MMOs.push_back(MMO);
5836 }
5837
5838 Result = DAG.getMemIntrinsicNode(Info->opc, getCurSDLoc(), VTs, Ops,
5839 Info->memVT, MMOs);
5840 } else {
5841 Result = getTargetNonMemIntrinsicNode(*I.getType(), HasChain, Ops, VTs);
5842 }
5843
5844 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
5845
5846 setValue(&I, Result);
5847}
5848
5849/// GetSignificand - Get the significand and build it into a floating-point
5850/// number with exponent of 1:
5851///
5852/// Op = (Op & 0x007fffff) | 0x3f800000;
5853///
5854/// where Op is the hexadecimal representation of floating point value.
5856 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5857 DAG.getConstant(0x007fffff, dl, MVT::i32));
5858 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5859 DAG.getConstant(0x3f800000, dl, MVT::i32));
5860 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5861}
5862
5863/// GetExponent - Get the exponent:
5864///
5865/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5866///
5867/// where Op is the hexadecimal representation of floating point value.
5869 const TargetLowering &TLI, const SDLoc &dl) {
5870 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5871 DAG.getConstant(0x7f800000, dl, MVT::i32));
5872 SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
5873 DAG.getShiftAmountConstant(23, MVT::i32, dl));
5874 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5875 DAG.getConstant(127, dl, MVT::i32));
5876 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5877}
5878
5879/// getF32Constant - Get 32-bit floating point constant.
5880static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
5881 const SDLoc &dl) {
5882 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5883 MVT::f32);
5884}
5885
5887 SelectionDAG &DAG) {
5888 // TODO: What fast-math-flags should be set on the floating-point nodes?
5889
5890 // IntegerPartOfX = ((int32_t)(t0);
5891 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5892
5893 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5894 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5895 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5896
5897 // IntegerPartOfX <<= 23;
5898 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5899 DAG.getShiftAmountConstant(23, MVT::i32, dl));
5900
5901 SDValue TwoToFractionalPartOfX;
5902 if (LimitFloatPrecision <= 6) {
5903 // For floating-point precision of 6:
5904 //
5905 // TwoToFractionalPartOfX =
5906 // 0.997535578f +
5907 // (0.735607626f + 0.252464424f * x) * x;
5908 //
5909 // error 0.0144103317, which is 6 bits
5910 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5911 getF32Constant(DAG, 0x3e814304, dl));
5912 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5913 getF32Constant(DAG, 0x3f3c50c8, dl));
5914 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5915 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5916 getF32Constant(DAG, 0x3f7f5e7e, dl));
5917 } else if (LimitFloatPrecision <= 12) {
5918 // For floating-point precision of 12:
5919 //
5920 // TwoToFractionalPartOfX =
5921 // 0.999892986f +
5922 // (0.696457318f +
5923 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5924 //
5925 // error 0.000107046256, which is 13 to 14 bits
5926 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5927 getF32Constant(DAG, 0x3da235e3, dl));
5928 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5929 getF32Constant(DAG, 0x3e65b8f3, dl));
5930 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5931 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5932 getF32Constant(DAG, 0x3f324b07, dl));
5933 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5934 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5935 getF32Constant(DAG, 0x3f7ff8fd, dl));
5936 } else { // LimitFloatPrecision <= 18
5937 // For floating-point precision of 18:
5938 //
5939 // TwoToFractionalPartOfX =
5940 // 0.999999982f +
5941 // (0.693148872f +
5942 // (0.240227044f +
5943 // (0.554906021e-1f +
5944 // (0.961591928e-2f +
5945 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5946 // error 2.47208000*10^(-7), which is better than 18 bits
5947 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5948 getF32Constant(DAG, 0x3924b03e, dl));
5949 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5950 getF32Constant(DAG, 0x3ab24b87, dl));
5951 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5952 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5953 getF32Constant(DAG, 0x3c1d8c17, dl));
5954 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5955 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5956 getF32Constant(DAG, 0x3d634a1d, dl));
5957 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5958 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5959 getF32Constant(DAG, 0x3e75fe14, dl));
5960 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5961 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5962 getF32Constant(DAG, 0x3f317234, dl));
5963 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5964 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5965 getF32Constant(DAG, 0x3f800000, dl));
5966 }
5967
5968 // Add the exponent into the result in integer domain.
5969 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5970 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5971 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5972}
5973
5974/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5975/// limited-precision mode.
5977 const TargetLowering &TLI, SDNodeFlags Flags) {
5978 if (Op.getValueType() == MVT::f32 &&
5980
5981 // Put the exponent in the right bit position for later addition to the
5982 // final result:
5983 //
5984 // t0 = Op * log2(e)
5985
5986 // TODO: What fast-math-flags should be set here?
5987 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5988 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5989 return getLimitedPrecisionExp2(t0, dl, DAG);
5990 }
5991
5992 // No special expansion.
5993 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5994}
5995
5996/// expandLog - Lower a log intrinsic. Handles the special sequences for
5997/// limited-precision mode.
5999 const TargetLowering &TLI, SDNodeFlags Flags) {
6000 // TODO: What fast-math-flags should be set on the floating-point nodes?
6001
6002 if (Op.getValueType() == MVT::f32 &&
6004 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
6005
6006 // Scale the exponent by log(2).
6007 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
6008 SDValue LogOfExponent =
6009 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
6010 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
6011
6012 // Get the significand and build it into a floating-point number with
6013 // exponent of 1.
6014 SDValue X = GetSignificand(DAG, Op1, dl);
6015
6016 SDValue LogOfMantissa;
6017 if (LimitFloatPrecision <= 6) {
6018 // For floating-point precision of 6:
6019 //
6020 // LogofMantissa =
6021 // -1.1609546f +
6022 // (1.4034025f - 0.23903021f * x) * x;
6023 //
6024 // error 0.0034276066, which is better than 8 bits
6025 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6026 getF32Constant(DAG, 0xbe74c456, dl));
6027 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6028 getF32Constant(DAG, 0x3fb3a2b1, dl));
6029 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6030 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6031 getF32Constant(DAG, 0x3f949a29, dl));
6032 } else if (LimitFloatPrecision <= 12) {
6033 // For floating-point precision of 12:
6034 //
6035 // LogOfMantissa =
6036 // -1.7417939f +
6037 // (2.8212026f +
6038 // (-1.4699568f +
6039 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
6040 //
6041 // error 0.000061011436, which is 14 bits
6042 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6043 getF32Constant(DAG, 0xbd67b6d6, dl));
6044 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6045 getF32Constant(DAG, 0x3ee4f4b8, dl));
6046 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6047 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6048 getF32Constant(DAG, 0x3fbc278b, dl));
6049 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6050 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6051 getF32Constant(DAG, 0x40348e95, dl));
6052 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6053 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6054 getF32Constant(DAG, 0x3fdef31a, dl));
6055 } else { // LimitFloatPrecision <= 18
6056 // For floating-point precision of 18:
6057 //
6058 // LogOfMantissa =
6059 // -2.1072184f +
6060 // (4.2372794f +
6061 // (-3.7029485f +
6062 // (2.2781945f +
6063 // (-0.87823314f +
6064 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
6065 //
6066 // error 0.0000023660568, which is better than 18 bits
6067 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6068 getF32Constant(DAG, 0xbc91e5ac, dl));
6069 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6070 getF32Constant(DAG, 0x3e4350aa, dl));
6071 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6072 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6073 getF32Constant(DAG, 0x3f60d3e3, dl));
6074 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6075 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6076 getF32Constant(DAG, 0x4011cdf0, dl));
6077 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6078 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6079 getF32Constant(DAG, 0x406cfd1c, dl));
6080 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6081 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
6082 getF32Constant(DAG, 0x408797cb, dl));
6083 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
6084 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
6085 getF32Constant(DAG, 0x4006dcab, dl));
6086 }
6087
6088 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
6089 }
6090
6091 // No special expansion.
6092 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
6093}
6094
6095/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
6096/// limited-precision mode.
6098 const TargetLowering &TLI, SDNodeFlags Flags) {
6099 // TODO: What fast-math-flags should be set on the floating-point nodes?
6100
6101 if (Op.getValueType() == MVT::f32 &&
6103 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
6104
6105 // Get the exponent.
6106 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
6107
6108 // Get the significand and build it into a floating-point number with
6109 // exponent of 1.
6110 SDValue X = GetSignificand(DAG, Op1, dl);
6111
6112 // Different possible minimax approximations of significand in
6113 // floating-point for various degrees of accuracy over [1,2].
6114 SDValue Log2ofMantissa;
6115 if (LimitFloatPrecision <= 6) {
6116 // For floating-point precision of 6:
6117 //
6118 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
6119 //
6120 // error 0.0049451742, which is more than 7 bits
6121 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6122 getF32Constant(DAG, 0xbeb08fe0, dl));
6123 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6124 getF32Constant(DAG, 0x40019463, dl));
6125 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6126 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6127 getF32Constant(DAG, 0x3fd6633d, dl));
6128 } else if (LimitFloatPrecision <= 12) {
6129 // For floating-point precision of 12:
6130 //
6131 // Log2ofMantissa =
6132 // -2.51285454f +
6133 // (4.07009056f +
6134 // (-2.12067489f +
6135 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
6136 //
6137 // error 0.0000876136000, which is better than 13 bits
6138 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6139 getF32Constant(DAG, 0xbda7262e, dl));
6140 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6141 getF32Constant(DAG, 0x3f25280b, dl));
6142 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6143 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6144 getF32Constant(DAG, 0x4007b923, dl));
6145 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6146 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6147 getF32Constant(DAG, 0x40823e2f, dl));
6148 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6149 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6150 getF32Constant(DAG, 0x4020d29c, dl));
6151 } else { // LimitFloatPrecision <= 18
6152 // For floating-point precision of 18:
6153 //
6154 // Log2ofMantissa =
6155 // -3.0400495f +
6156 // (6.1129976f +
6157 // (-5.3420409f +
6158 // (3.2865683f +
6159 // (-1.2669343f +
6160 // (0.27515199f -
6161 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
6162 //
6163 // error 0.0000018516, which is better than 18 bits
6164 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6165 getF32Constant(DAG, 0xbcd2769e, dl));
6166 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6167 getF32Constant(DAG, 0x3e8ce0b9, dl));
6168 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6169 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6170 getF32Constant(DAG, 0x3fa22ae7, dl));
6171 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6172 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6173 getF32Constant(DAG, 0x40525723, dl));
6174 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6175 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6176 getF32Constant(DAG, 0x40aaf200, dl));
6177 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6178 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
6179 getF32Constant(DAG, 0x40c39dad, dl));
6180 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
6181 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
6182 getF32Constant(DAG, 0x4042902c, dl));
6183 }
6184
6185 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
6186 }
6187
6188 // No special expansion.
6189 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
6190}
6191
6192/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
6193/// limited-precision mode.
6195 const TargetLowering &TLI, SDNodeFlags Flags) {
6196 // TODO: What fast-math-flags should be set on the floating-point nodes?
6197
6198 if (Op.getValueType() == MVT::f32 &&
6200 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
6201
6202 // Scale the exponent by log10(2) [0.30102999f].
6203 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
6204 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
6205 getF32Constant(DAG, 0x3e9a209a, dl));
6206
6207 // Get the significand and build it into a floating-point number with
6208 // exponent of 1.
6209 SDValue X = GetSignificand(DAG, Op1, dl);
6210
6211 SDValue Log10ofMantissa;
6212 if (LimitFloatPrecision <= 6) {
6213 // For floating-point precision of 6:
6214 //
6215 // Log10ofMantissa =
6216 // -0.50419619f +
6217 // (0.60948995f - 0.10380950f * x) * x;
6218 //
6219 // error 0.0014886165, which is 6 bits
6220 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6221 getF32Constant(DAG, 0xbdd49a13, dl));
6222 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6223 getF32Constant(DAG, 0x3f1c0789, dl));
6224 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6225 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6226 getF32Constant(DAG, 0x3f011300, dl));
6227 } else if (LimitFloatPrecision <= 12) {
6228 // For floating-point precision of 12:
6229 //
6230 // Log10ofMantissa =
6231 // -0.64831180f +
6232 // (0.91751397f +
6233 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
6234 //
6235 // error 0.00019228036, which is better than 12 bits
6236 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6237 getF32Constant(DAG, 0x3d431f31, dl));
6238 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
6239 getF32Constant(DAG, 0x3ea21fb2, dl));
6240 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6241 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
6242 getF32Constant(DAG, 0x3f6ae232, dl));
6243 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6244 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
6245 getF32Constant(DAG, 0x3f25f7c3, dl));
6246 } else { // LimitFloatPrecision <= 18
6247 // For floating-point precision of 18:
6248 //
6249 // Log10ofMantissa =
6250 // -0.84299375f +
6251 // (1.5327582f +
6252 // (-1.0688956f +
6253 // (0.49102474f +
6254 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
6255 //
6256 // error 0.0000037995730, which is better than 18 bits
6257 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6258 getF32Constant(DAG, 0x3c5d51ce, dl));
6259 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
6260 getF32Constant(DAG, 0x3e00685a, dl));
6261 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6262 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
6263 getF32Constant(DAG, 0x3efb6798, dl));
6264 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6265 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
6266 getF32Constant(DAG, 0x3f88d192, dl));
6267 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6268 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
6269 getF32Constant(DAG, 0x3fc4316c, dl));
6270 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6271 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
6272 getF32Constant(DAG, 0x3f57ce70, dl));
6273 }
6274
6275 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
6276 }
6277
6278 // No special expansion.
6279 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
6280}
6281
6282/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
6283/// limited-precision mode.
6285 const TargetLowering &TLI, SDNodeFlags Flags) {
6286 if (Op.getValueType() == MVT::f32 &&
6288 return getLimitedPrecisionExp2(Op, dl, DAG);
6289
6290 // No special expansion.
6291 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
6292}
6293
6294/// visitPow - Lower a pow intrinsic. Handles the special sequences for
6295/// limited-precision mode with x == 10.0f.
6297 SelectionDAG &DAG, const TargetLowering &TLI,
6298 SDNodeFlags Flags) {
6299 bool IsExp10 = false;
6300 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
6303 APFloat Ten(10.0f);
6304 IsExp10 = LHSC->isExactlyValue(Ten);
6305 }
6306 }
6307
6308 // TODO: What fast-math-flags should be set on the FMUL node?
6309 if (IsExp10) {
6310 // Put the exponent in the right bit position for later addition to the
6311 // final result:
6312 //
6313 // #define LOG2OF10 3.3219281f
6314 // t0 = Op * LOG2OF10;
6315 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
6316 getF32Constant(DAG, 0x40549a78, dl));
6317 return getLimitedPrecisionExp2(t0, dl, DAG);
6318 }
6319
6320 // No special expansion.
6321 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
6322}
6323
6324/// ExpandPowI - Expand a llvm.powi intrinsic.
6326 SelectionDAG &DAG) {
6327 // If RHS is a constant, we can expand this out to a multiplication tree if
6328 // it's beneficial on the target, otherwise we end up lowering to a call to
6329 // __powidf2 (for example).
6331 unsigned Val = RHSC->getSExtValue();
6332
6333 // powi(x, 0) -> 1.0
6334 if (Val == 0)
6335 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
6336
6338 Val, DAG.shouldOptForSize())) {
6339 // Get the exponent as a positive value.
6340 if ((int)Val < 0)
6341 Val = -Val;
6342 // We use the simple binary decomposition method to generate the multiply
6343 // sequence. There are more optimal ways to do this (for example,
6344 // powi(x,15) generates one more multiply than it should), but this has
6345 // the benefit of being both really simple and much better than a libcall.
6346 SDValue Res; // Logically starts equal to 1.0
6347 SDValue CurSquare = LHS;
6348 // TODO: Intrinsics should have fast-math-flags that propagate to these
6349 // nodes.
6350 while (Val) {
6351 if (Val & 1) {
6352 if (Res.getNode())
6353 Res =
6354 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
6355 else
6356 Res = CurSquare; // 1.0*CurSquare.
6357 }
6358
6359 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
6360 CurSquare, CurSquare);
6361 Val >>= 1;
6362 }
6363
6364 // If the original was negative, invert the result, producing 1/(x*x*x).
6365 if (RHSC->getSExtValue() < 0)
6366 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
6367 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
6368 return Res;
6369 }
6370 }
6371
6372 // Otherwise, expand to a libcall.
6373 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
6374}
6375
6376static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
6377 SDValue LHS, SDValue RHS, SDValue Scale,
6378 SelectionDAG &DAG, const TargetLowering &TLI) {
6379 EVT VT = LHS.getValueType();
6380 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
6381 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
6382 LLVMContext &Ctx = *DAG.getContext();
6383
6384 // If the type is legal but the operation isn't, this node might survive all
6385 // the way to operation legalization. If we end up there and we do not have
6386 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
6387 // node.
6388
6389 // Coax the legalizer into expanding the node during type legalization instead
6390 // by bumping the size by one bit. This will force it to Promote, enabling the
6391 // early expansion and avoiding the need to expand later.
6392
6393 // We don't have to do this if Scale is 0; that can always be expanded, unless
6394 // it's a saturating signed operation. Those can experience true integer
6395 // division overflow, a case which we must avoid.
6396
6397 // FIXME: We wouldn't have to do this (or any of the early
6398 // expansion/promotion) if it was possible to expand a libcall of an
6399 // illegal type during operation legalization. But it's not, so things
6400 // get a bit hacky.
6401 unsigned ScaleInt = Scale->getAsZExtVal();
6402 if ((ScaleInt > 0 || (Saturating && Signed)) &&
6403 (TLI.isTypeLegal(VT) ||
6404 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
6405 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
6406 Opcode, VT, ScaleInt);
6407 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
6408 EVT PromVT;
6409 if (VT.isScalarInteger())
6410 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
6411 else if (VT.isVector()) {
6412 PromVT = VT.getVectorElementType();
6413 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
6414 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
6415 } else
6416 llvm_unreachable("Wrong VT for DIVFIX?");
6417 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
6418 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
6419 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
6420 // For saturating operations, we need to shift up the LHS to get the
6421 // proper saturation width, and then shift down again afterwards.
6422 if (Saturating)
6423 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
6424 DAG.getConstant(1, DL, ShiftTy));
6425 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
6426 if (Saturating)
6427 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
6428 DAG.getConstant(1, DL, ShiftTy));
6429 return DAG.getZExtOrTrunc(Res, DL, VT);
6430 }
6431 }
6432
6433 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
6434}
6435
6436// getUnderlyingArgRegs - Find underlying registers used for a truncated,
6437// bitcasted, or split argument. Returns a list of <Register, size in bits>
6438static void
6439getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
6440 const SDValue &N) {
6441 switch (N.getOpcode()) {
6442 case ISD::CopyFromReg: {
6443 SDValue Op = N.getOperand(1);
6444 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
6445 Op.getValueType().getSizeInBits());
6446 return;
6447 }
6448 case ISD::BITCAST:
6449 case ISD::AssertZext:
6450 case ISD::AssertSext:
6451 case ISD::TRUNCATE:
6452 getUnderlyingArgRegs(Regs, N.getOperand(0));
6453 return;
6454 case ISD::BUILD_PAIR:
6455 case ISD::BUILD_VECTOR:
6457 for (SDValue Op : N->op_values())
6458 getUnderlyingArgRegs(Regs, Op);
6459 return;
6460 default:
6461 return;
6462 }
6463}
6464
6465/// If the DbgValueInst is a dbg_value of a function argument, create the
6466/// corresponding DBG_VALUE machine instruction for it now. At the end of
6467/// instruction selection, they will be inserted to the entry BB.
6468/// We don't currently support this for variadic dbg_values, as they shouldn't
6469/// appear for function arguments or in the prologue.
6470bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6471 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6472 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6473 const Argument *Arg = dyn_cast<Argument>(V);
6474 if (!Arg)
6475 return false;
6476
6477 MachineFunction &MF = DAG.getMachineFunction();
6478 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6479
6480 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6481 // we've been asked to pursue.
6482 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6483 bool Indirect) {
6484 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6485 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6486 // pointing at the VReg, which will be patched up later.
6487 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
6489 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6490 /* isKill */ false, /* isDead */ false,
6491 /* isUndef */ false, /* isEarlyClobber */ false,
6492 /* SubReg */ 0, /* isDebug */ true)});
6493
6494 auto *NewDIExpr = FragExpr;
6495 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6496 // the DIExpression.
6497 if (Indirect)
6498 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
6500 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
6501 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
6502 } else {
6503 // Create a completely standard DBG_VALUE.
6504 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
6505 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
6506 }
6507 };
6508
6509 if (Kind == FuncArgumentDbgValueKind::Value) {
6510 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6511 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6512 // the entry block.
6513 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6514 if (!IsInEntryBlock)
6515 return false;
6516
6517 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6518 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6519 // variable that also is a param.
6520 //
6521 // Although, if we are at the top of the entry block already, we can still
6522 // emit using ArgDbgValue. This might catch some situations when the
6523 // dbg.value refers to an argument that isn't used in the entry block, so
6524 // any CopyToReg node would be optimized out and the only way to express
6525 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6526 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6527 // we should only emit as ArgDbgValue if the Variable is an argument to the
6528 // current function, and the dbg.value intrinsic is found in the entry
6529 // block.
6530 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6531 !DL->getInlinedAt();
6532 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6533 if (!IsInPrologue && !VariableIsFunctionInputArg)
6534 return false;
6535
6536 // Here we assume that a function argument on IR level only can be used to
6537 // describe one input parameter on source level. If we for example have
6538 // source code like this
6539 //
6540 // struct A { long x, y; };
6541 // void foo(struct A a, long b) {
6542 // ...
6543 // b = a.x;
6544 // ...
6545 // }
6546 //
6547 // and IR like this
6548 //
6549 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6550 // entry:
6551 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6552 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6553 // call void @llvm.dbg.value(metadata i32 %b, "b",
6554 // ...
6555 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6556 // ...
6557 //
6558 // then the last dbg.value is describing a parameter "b" using a value that
6559 // is an argument. But since we already has used %a1 to describe a parameter
6560 // we should not handle that last dbg.value here (that would result in an
6561 // incorrect hoisting of the DBG_VALUE to the function entry).
6562 // Notice that we allow one dbg.value per IR level argument, to accommodate
6563 // for the situation with fragments above.
6564 // If there is no node for the value being handled, we return true to skip
6565 // the normal generation of debug info, as it would kill existing debug
6566 // info for the parameter in case of duplicates.
6567 if (VariableIsFunctionInputArg) {
6568 unsigned ArgNo = Arg->getArgNo();
6569 if (ArgNo >= FuncInfo.DescribedArgs.size())
6570 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6571 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6572 return !NodeMap[V].getNode();
6573 FuncInfo.DescribedArgs.set(ArgNo);
6574 }
6575 }
6576
6577 bool IsIndirect = false;
6578 std::optional<MachineOperand> Op;
6579 // Some arguments' frame index is recorded during argument lowering.
6580 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6581 if (FI != std::numeric_limits<int>::max())
6583
6585 if (!Op && N.getNode()) {
6586 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6587 Register Reg;
6588 if (ArgRegsAndSizes.size() == 1)
6589 Reg = ArgRegsAndSizes.front().first;
6590
6591 if (Reg && Reg.isVirtual()) {
6592 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6593 Register PR = RegInfo.getLiveInPhysReg(Reg);
6594 if (PR)
6595 Reg = PR;
6596 }
6597 if (Reg) {
6599 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6600 }
6601 }
6602
6603 if (!Op && N.getNode()) {
6604 // Check if frame index is available.
6605 SDValue LCandidate = peekThroughBitcasts(N);
6606 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6607 if (FrameIndexSDNode *FINode =
6608 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6609 Op = MachineOperand::CreateFI(FINode->getIndex());
6610 }
6611
6612 if (!Op) {
6613 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6614 auto splitMultiRegDbgValue =
6615 [&](ArrayRef<std::pair<Register, TypeSize>> SplitRegs) -> bool {
6616 unsigned Offset = 0;
6617 for (const auto &[Reg, RegSizeInBits] : SplitRegs) {
6618 // FIXME: Scalable sizes are not supported in fragment expressions.
6619 if (RegSizeInBits.isScalable())
6620 return false;
6621
6622 // If the expression is already a fragment, the current register
6623 // offset+size might extend beyond the fragment. In this case, only
6624 // the register bits that are inside the fragment are relevant.
6625 int RegFragmentSizeInBits = RegSizeInBits.getFixedValue();
6626 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6627 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6628 // The register is entirely outside the expression fragment,
6629 // so is irrelevant for debug info.
6630 if (Offset >= ExprFragmentSizeInBits)
6631 break;
6632 // The register is partially outside the expression fragment, only
6633 // the low bits within the fragment are relevant for debug info.
6634 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6635 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6636 }
6637 }
6638
6639 auto FragmentExpr = DIExpression::createFragmentExpression(
6640 Expr, Offset, RegFragmentSizeInBits);
6641 Offset += RegSizeInBits.getFixedValue();
6642 // If a valid fragment expression cannot be created, the variable's
6643 // correct value cannot be determined and so it is set as poison.
6644 if (!FragmentExpr) {
6645 SDDbgValue *SDV = DAG.getConstantDbgValue(
6646 Variable, Expr, PoisonValue::get(V->getType()), DL, SDNodeOrder);
6647 DAG.AddDbgValue(SDV, false);
6648 continue;
6649 }
6650 MachineInstr *NewMI = MakeVRegDbgValue(
6651 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6652 FuncInfo.ArgDbgValues.push_back(NewMI);
6653 }
6654
6655 return true;
6656 };
6657
6658 // Check if ValueMap has reg number.
6660 VMI = FuncInfo.ValueMap.find(V);
6661 if (VMI != FuncInfo.ValueMap.end()) {
6662 const auto &TLI = DAG.getTargetLoweringInfo();
6663 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6664 V->getType(), std::nullopt);
6665 if (RFV.occupiesMultipleRegs())
6666 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6667
6668 Op = MachineOperand::CreateReg(VMI->second, false);
6669 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6670 } else if (ArgRegsAndSizes.size() > 1) {
6671 // This was split due to the calling convention, and no virtual register
6672 // mapping exists for the value.
6673 return splitMultiRegDbgValue(ArgRegsAndSizes);
6674 }
6675 }
6676
6677 if (!Op)
6678 return false;
6679
6680 assert(Variable->isValidLocationForIntrinsic(DL) &&
6681 "Expected inlined-at fields to agree");
6682 MachineInstr *NewMI = nullptr;
6683
6684 if (Op->isReg())
6685 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6686 else
6687 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6688 Variable, Expr);
6689
6690 // Otherwise, use ArgDbgValues.
6691 FuncInfo.ArgDbgValues.push_back(NewMI);
6692 return true;
6693}
6694
6695/// Return the appropriate SDDbgValue based on N.
6696SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6697 DILocalVariable *Variable,
6698 DIExpression *Expr,
6699 const DebugLoc &dl,
6700 unsigned DbgSDNodeOrder) {
6701 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6702 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6703 // stack slot locations.
6704 //
6705 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6706 // debug values here after optimization:
6707 //
6708 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6709 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6710 //
6711 // Both describe the direct values of their associated variables.
6712 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6713 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6714 }
6715 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6716 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6717}
6718
6719static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6720 switch (Intrinsic) {
6721 case Intrinsic::smul_fix:
6722 return ISD::SMULFIX;
6723 case Intrinsic::umul_fix:
6724 return ISD::UMULFIX;
6725 case Intrinsic::smul_fix_sat:
6726 return ISD::SMULFIXSAT;
6727 case Intrinsic::umul_fix_sat:
6728 return ISD::UMULFIXSAT;
6729 case Intrinsic::sdiv_fix:
6730 return ISD::SDIVFIX;
6731 case Intrinsic::udiv_fix:
6732 return ISD::UDIVFIX;
6733 case Intrinsic::sdiv_fix_sat:
6734 return ISD::SDIVFIXSAT;
6735 case Intrinsic::udiv_fix_sat:
6736 return ISD::UDIVFIXSAT;
6737 default:
6738 llvm_unreachable("Unhandled fixed point intrinsic");
6739 }
6740}
6741
6742/// Given a @llvm.call.preallocated.setup, return the corresponding
6743/// preallocated call.
6744static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6745 assert(cast<CallBase>(PreallocatedSetup)
6747 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6748 "expected call_preallocated_setup Value");
6749 for (const auto *U : PreallocatedSetup->users()) {
6750 auto *UseCall = cast<CallBase>(U);
6751 const Function *Fn = UseCall->getCalledFunction();
6752 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6753 return UseCall;
6754 }
6755 }
6756 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6757}
6758
6759/// If DI is a debug value with an EntryValue expression, lower it using the
6760/// corresponding physical register of the associated Argument value
6761/// (guaranteed to exist by the verifier).
6762bool SelectionDAGBuilder::visitEntryValueDbgValue(
6764 DIExpression *Expr, DebugLoc DbgLoc) {
6765 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6766 return false;
6767
6768 // These properties are guaranteed by the verifier.
6769 const Argument *Arg = cast<Argument>(Values[0]);
6770 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6771
6772 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6773 if (ArgIt == FuncInfo.ValueMap.end()) {
6774 LLVM_DEBUG(
6775 dbgs() << "Dropping dbg.value: expression is entry_value but "
6776 "couldn't find an associated register for the Argument\n");
6777 return true;
6778 }
6779 Register ArgVReg = ArgIt->getSecond();
6780
6781 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6782 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6783 SDDbgValue *SDV = DAG.getVRegDbgValue(
6784 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6785 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6786 return true;
6787 }
6788 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6789 "couldn't find a physical register\n");
6790 return true;
6791}
6792
6793/// Lower the call to the specified intrinsic function.
6794void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6795 unsigned Intrinsic) {
6796 SDLoc sdl = getCurSDLoc();
6797 switch (Intrinsic) {
6798 case Intrinsic::experimental_convergence_anchor:
6799 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6800 break;
6801 case Intrinsic::experimental_convergence_entry:
6802 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6803 break;
6804 case Intrinsic::experimental_convergence_loop: {
6805 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6806 auto *Token = Bundle->Inputs[0].get();
6807 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6808 getValue(Token)));
6809 break;
6810 }
6811 }
6812}
6813
6814void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6815 unsigned IntrinsicID) {
6816 // For now, we're only lowering an 'add' histogram.
6817 // We can add others later, e.g. saturating adds, min/max.
6818 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6819 "Tried to lower unsupported histogram type");
6820 SDLoc sdl = getCurSDLoc();
6821 Value *Ptr = I.getOperand(0);
6822 SDValue Inc = getValue(I.getOperand(1));
6823 SDValue Mask = getValue(I.getOperand(2));
6824
6825 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6826 EVT VT = Inc.getValueType();
6827 Align Alignment = DAG.getEVTAlign(VT);
6828
6829 const MDNode *Ranges = getRangeMetadata(I);
6830
6831 SDValue Root = DAG.getRoot();
6832 SDValue Base;
6833 SDValue Index;
6834 SDValue Scale;
6835 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
6836 I.getParent(), VT.getScalarStoreSize());
6837
6838 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6839
6840 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6841 MachinePointerInfo(AS),
6843 MemoryLocation::UnknownSize, Alignment,
6844 MMOMetadata(I.getAAMetadata(), Ranges));
6845
6846 if (!UniformBase) {
6847 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
6848 Index = getValue(Ptr);
6849 Scale =
6850 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
6851 }
6852
6853 EVT IdxVT = Index.getValueType();
6854
6855 // Avoid using e.g. i32 as index type when the increment must be performed
6856 // on i64's.
6857 bool MustExtendIndex = VT.getScalarSizeInBits() > IdxVT.getScalarSizeInBits();
6858 EVT EltTy = MustExtendIndex ? VT : IdxVT.getVectorElementType();
6859 if (MustExtendIndex || TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
6860 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
6861 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
6862 }
6863
6864 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6865
6866 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6867 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
6868 Ops, MMO, ISD::SIGNED_SCALED);
6869
6870 setValue(&I, Histogram);
6871 DAG.setRoot(Histogram);
6872}
6873
6874void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6875 unsigned Intrinsic) {
6876 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6877 "Tried lowering invalid vector extract last");
6878 SDLoc sdl = getCurSDLoc();
6879 const DataLayout &Layout = DAG.getDataLayout();
6880 SDValue Data = getValue(I.getOperand(0));
6881 SDValue Mask = getValue(I.getOperand(1));
6882
6883 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6884 EVT ResVT = TLI.getValueType(Layout, I.getType());
6885
6886 EVT ExtVT = TLI.getVectorIdxTy(Layout);
6887 SDValue Idx = DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, sdl, ExtVT, Mask);
6888 SDValue Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl, ResVT, Data, Idx);
6889
6890 Value *Default = I.getOperand(2);
6892 SDValue PassThru = getValue(Default);
6893 EVT BoolVT = Mask.getValueType().getScalarType();
6894 SDValue AnyActive = DAG.getNode(ISD::VECREDUCE_OR, sdl, BoolVT, Mask);
6895 Result = DAG.getSelect(sdl, ResVT, AnyActive, Result, PassThru);
6896 }
6897
6898 setValue(&I, Result);
6899}
6900
6901/// Lower the call to the specified intrinsic function.
6902void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6903 unsigned Intrinsic) {
6904 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6905 SDLoc sdl = getCurSDLoc();
6906 DebugLoc dl = getCurDebugLoc();
6907 SDValue Res;
6908
6909 SDNodeFlags Flags;
6910 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6911 Flags.copyFMF(*FPOp);
6912
6913 switch (Intrinsic) {
6914 default:
6915 // By default, turn this into a target intrinsic node.
6916 visitTargetIntrinsic(I, Intrinsic);
6917 return;
6918 case Intrinsic::vscale: {
6919 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6920 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6921 return;
6922 }
6923 case Intrinsic::vastart: visitVAStart(I); return;
6924 case Intrinsic::vaend: visitVAEnd(I); return;
6925 case Intrinsic::vacopy: visitVACopy(I); return;
6926 case Intrinsic::returnaddress:
6927 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
6928 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6929 getValue(I.getArgOperand(0))));
6930 return;
6931 case Intrinsic::addressofreturnaddress:
6932 setValue(&I,
6933 DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
6934 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6935 return;
6936 case Intrinsic::sponentry:
6937 setValue(&I,
6938 DAG.getNode(ISD::SPONENTRY, sdl,
6939 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6940 return;
6941 case Intrinsic::frameaddress:
6942 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
6943 TLI.getFrameIndexTy(DAG.getDataLayout()),
6944 getValue(I.getArgOperand(0))));
6945 return;
6946 case Intrinsic::read_volatile_register:
6947 case Intrinsic::read_register: {
6948 Value *Reg = I.getArgOperand(0);
6949 SDValue Chain = getRoot();
6951 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6952 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6953 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6954 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6955 setValue(&I, Res);
6956 DAG.setRoot(Res.getValue(1));
6957 return;
6958 }
6959 case Intrinsic::write_register: {
6960 Value *Reg = I.getArgOperand(0);
6961 Value *RegValue = I.getArgOperand(1);
6962 SDValue Chain = getRoot();
6964 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6965 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6966 RegName, getValue(RegValue)));
6967 return;
6968 }
6969 case Intrinsic::write_volatile_register: {
6970 Value *Reg = I.getArgOperand(0);
6971 Value *RegValue = I.getArgOperand(1);
6972 SDValue Chain = getRoot();
6973 const MDNode *MD = cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata());
6974 SDValue RegName = DAG.getMDNode(MD);
6975 EVT VT = TLI.getValueType(DAG.getDataLayout(), RegValue->getType());
6976 SDValue WriteChain = DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other,
6977 Chain, RegName, getValue(RegValue));
6978 // FAKE_USE of the physical register marks it live after the WRITE_REGISTER,
6979 // preventing the backend from dead-eliminating the write. This is
6980 // preferred over READ_REGISTER, which would emit extra register copies
6981 // (e.g. fmov xN, dN for FP/SIMD registers).
6982 const MDString *RegStr = cast<MDString>(MD->getOperand(0));
6983 LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
6984 const MachineFunction &MF = DAG.getMachineFunction();
6985 Register PhysReg =
6986 TLI.getRegisterByName(RegStr->getString().data(), Ty, MF);
6987 if (PhysReg.isValid()) {
6988 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6989 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(PhysReg);
6990 MVT RegVT = *TRI->legalclasstypes_begin(*RC);
6991 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other,
6992 {WriteChain, DAG.getRegister(PhysReg, RegVT)}));
6993 } else {
6994 DAG.setRoot(WriteChain);
6995 }
6996 return;
6997 }
6998 case Intrinsic::memcpy:
6999 case Intrinsic::memcpy_inline: {
7000 const auto &MCI = cast<MemCpyInst>(I);
7001 SDValue Dst = getValue(I.getArgOperand(0));
7002 SDValue Src = getValue(I.getArgOperand(1));
7003 SDValue Size = getValue(I.getArgOperand(2));
7004 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
7005 "memcpy_inline needs constant size");
7006 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
7007 Align DstAlign = MCI.getDestAlign().valueOrOne();
7008 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
7009 bool isVol = MCI.isVolatile();
7010 SDValue Root = isVol ? getRoot() : getMemoryRoot();
7011 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, DstAlign, SrcAlign,
7012 isVol, MCI.isForceInlined(), &I, std::nullopt,
7013 MachinePointerInfo(I.getArgOperand(0)),
7014 MachinePointerInfo(I.getArgOperand(1)),
7015 I.getAAMetadata(), BatchAA);
7016 updateDAGForMaybeTailCall(MC);
7017 return;
7018 }
7019 case Intrinsic::memset:
7020 case Intrinsic::memset_inline: {
7021 const auto &MSII = cast<MemSetInst>(I);
7022 SDValue Dst = getValue(I.getArgOperand(0));
7023 SDValue Value = getValue(I.getArgOperand(1));
7024 SDValue Size = getValue(I.getArgOperand(2));
7025 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
7026 "memset_inline needs constant size");
7027 // @llvm.memset defines 0 and 1 to both mean no alignment.
7028 Align DstAlign = MSII.getDestAlign().valueOrOne();
7029 bool isVol = MSII.isVolatile();
7030 SDValue Root = isVol ? getRoot() : getMemoryRoot();
7031 SDValue MC = DAG.getMemset(
7032 Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(),
7033 &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
7034 updateDAGForMaybeTailCall(MC);
7035 return;
7036 }
7037 case Intrinsic::memmove: {
7038 const auto &MMI = cast<MemMoveInst>(I);
7039 SDValue Op1 = getValue(I.getArgOperand(0));
7040 SDValue Op2 = getValue(I.getArgOperand(1));
7041 SDValue Op3 = getValue(I.getArgOperand(2));
7042 // @llvm.memmove defines 0 and 1 to both mean no alignment.
7043 Align DstAlign = MMI.getDestAlign().valueOrOne();
7044 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
7045 bool isVol = MMI.isVolatile();
7046 SDValue Root = isVol ? getRoot() : getMemoryRoot();
7047 SDValue MM = DAG.getMemmove(
7048 Root, sdl, Op1, Op2, Op3, DstAlign, SrcAlign, isVol, &I,
7049 /* OverrideTailCall */ std::nullopt,
7050 MachinePointerInfo(I.getArgOperand(0)),
7051 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), BatchAA);
7052 updateDAGForMaybeTailCall(MM);
7053 return;
7054 }
7055 case Intrinsic::memcpy_element_unordered_atomic: {
7056 auto &MI = cast<AnyMemCpyInst>(I);
7057 SDValue Dst = getValue(MI.getRawDest());
7058 SDValue Src = getValue(MI.getRawSource());
7059 SDValue Length = getValue(MI.getLength());
7060
7061 Type *LengthTy = MI.getLength()->getType();
7062 unsigned ElemSz = MI.getElementSizeInBytes();
7063 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7064 SDValue MC =
7065 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
7066 isTC, MachinePointerInfo(MI.getRawDest()),
7067 MachinePointerInfo(MI.getRawSource()));
7068 updateDAGForMaybeTailCall(MC);
7069 return;
7070 }
7071 case Intrinsic::memmove_element_unordered_atomic: {
7072 auto &MI = cast<AnyMemMoveInst>(I);
7073 SDValue Dst = getValue(MI.getRawDest());
7074 SDValue Src = getValue(MI.getRawSource());
7075 SDValue Length = getValue(MI.getLength());
7076
7077 Type *LengthTy = MI.getLength()->getType();
7078 unsigned ElemSz = MI.getElementSizeInBytes();
7079 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7080 SDValue MC =
7081 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
7082 isTC, MachinePointerInfo(MI.getRawDest()),
7083 MachinePointerInfo(MI.getRawSource()));
7084 updateDAGForMaybeTailCall(MC);
7085 return;
7086 }
7087 case Intrinsic::memset_element_unordered_atomic: {
7088 auto &MI = cast<AnyMemSetInst>(I);
7089 SDValue Dst = getValue(MI.getRawDest());
7090 SDValue Val = getValue(MI.getValue());
7091 SDValue Length = getValue(MI.getLength());
7092
7093 Type *LengthTy = MI.getLength()->getType();
7094 unsigned ElemSz = MI.getElementSizeInBytes();
7095 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7096 SDValue MC =
7097 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
7098 isTC, MachinePointerInfo(MI.getRawDest()));
7099 updateDAGForMaybeTailCall(MC);
7100 return;
7101 }
7102 case Intrinsic::call_preallocated_setup: {
7103 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
7104 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
7105 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
7106 getRoot(), SrcValue);
7107 setValue(&I, Res);
7108 DAG.setRoot(Res);
7109 return;
7110 }
7111 case Intrinsic::call_preallocated_arg: {
7112 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
7113 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
7114 SDValue Ops[3];
7115 Ops[0] = getRoot();
7116 Ops[1] = SrcValue;
7117 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7118 MVT::i32); // arg index
7119 SDValue Res = DAG.getNode(
7121 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
7122 setValue(&I, Res);
7123 DAG.setRoot(Res.getValue(1));
7124 return;
7125 }
7126
7127 case Intrinsic::eh_typeid_for: {
7128 // Find the type id for the given typeinfo.
7129 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
7130 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
7131 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
7132 setValue(&I, Res);
7133 return;
7134 }
7135
7136 case Intrinsic::eh_return_i32:
7137 case Intrinsic::eh_return_i64:
7138 DAG.getMachineFunction().setCallsEHReturn(true);
7139 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
7140 MVT::Other,
7142 getValue(I.getArgOperand(0)),
7143 getValue(I.getArgOperand(1))));
7144 return;
7145 case Intrinsic::eh_unwind_init:
7146 DAG.getMachineFunction().setCallsUnwindInit(true);
7147 return;
7148 case Intrinsic::eh_dwarf_cfa:
7149 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
7150 TLI.getPointerTy(DAG.getDataLayout()),
7151 getValue(I.getArgOperand(0))));
7152 return;
7153 case Intrinsic::eh_sjlj_callsite: {
7154 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
7155 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
7156
7157 FuncInfo.setCurrentCallSite(CI->getZExtValue());
7158 return;
7159 }
7160 case Intrinsic::eh_sjlj_functioncontext: {
7161 // Get and store the index of the function context.
7162 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7163 AllocaInst *FnCtx =
7164 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
7165 int FI = FuncInfo.StaticAllocaMap[FnCtx];
7167 return;
7168 }
7169 case Intrinsic::eh_sjlj_setjmp: {
7170 SDValue Ops[2];
7171 Ops[0] = getRoot();
7172 Ops[1] = getValue(I.getArgOperand(0));
7173 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
7174 DAG.getVTList(MVT::i32, MVT::Other), Ops);
7175 setValue(&I, Op.getValue(0));
7176 DAG.setRoot(Op.getValue(1));
7177 return;
7178 }
7179 case Intrinsic::eh_sjlj_longjmp:
7180 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
7181 getRoot(), getValue(I.getArgOperand(0))));
7182 return;
7183 case Intrinsic::eh_sjlj_setup_dispatch:
7184 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
7185 getRoot()));
7186 return;
7187 case Intrinsic::masked_gather:
7188 visitMaskedGather(I);
7189 return;
7190 case Intrinsic::masked_load:
7191 visitMaskedLoad(I);
7192 return;
7193 case Intrinsic::masked_scatter:
7194 visitMaskedScatter(I);
7195 return;
7196 case Intrinsic::masked_store:
7197 visitMaskedStore(I);
7198 return;
7199 case Intrinsic::masked_expandload:
7200 visitMaskedLoad(I, true /* IsExpanding */);
7201 return;
7202 case Intrinsic::masked_compressstore:
7203 visitMaskedStore(I, true /* IsCompressing */);
7204 return;
7205 case Intrinsic::speculative_load:
7206 visitSpeculativeLoad(I);
7207 return;
7208 case Intrinsic::powi:
7209 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
7210 getValue(I.getArgOperand(1)), DAG));
7211 return;
7212 case Intrinsic::log:
7213 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7214 return;
7215 case Intrinsic::log2:
7216 setValue(&I,
7217 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7218 return;
7219 case Intrinsic::log10:
7220 setValue(&I,
7221 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7222 return;
7223 case Intrinsic::exp:
7224 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7225 return;
7226 case Intrinsic::exp2:
7227 setValue(&I,
7228 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7229 return;
7230 case Intrinsic::pow:
7231 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
7232 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
7233 return;
7234 case Intrinsic::sqrt:
7235 case Intrinsic::fabs:
7236 case Intrinsic::sin:
7237 case Intrinsic::cos:
7238 case Intrinsic::tan:
7239 case Intrinsic::asin:
7240 case Intrinsic::acos:
7241 case Intrinsic::atan:
7242 case Intrinsic::sinh:
7243 case Intrinsic::cosh:
7244 case Intrinsic::tanh:
7245 case Intrinsic::exp10:
7246 case Intrinsic::floor:
7247 case Intrinsic::ceil:
7248 case Intrinsic::trunc:
7249 case Intrinsic::rint:
7250 case Intrinsic::nearbyint:
7251 case Intrinsic::round:
7252 case Intrinsic::roundeven:
7253 case Intrinsic::canonicalize: {
7254 unsigned Opcode;
7255 // clang-format off
7256 switch (Intrinsic) {
7257 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7258 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
7259 case Intrinsic::fabs: Opcode = ISD::FABS; break;
7260 case Intrinsic::sin: Opcode = ISD::FSIN; break;
7261 case Intrinsic::cos: Opcode = ISD::FCOS; break;
7262 case Intrinsic::tan: Opcode = ISD::FTAN; break;
7263 case Intrinsic::asin: Opcode = ISD::FASIN; break;
7264 case Intrinsic::acos: Opcode = ISD::FACOS; break;
7265 case Intrinsic::atan: Opcode = ISD::FATAN; break;
7266 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
7267 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
7268 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
7269 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
7270 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
7271 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
7272 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
7273 case Intrinsic::rint: Opcode = ISD::FRINT; break;
7274 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
7275 case Intrinsic::round: Opcode = ISD::FROUND; break;
7276 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
7277 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
7278 }
7279 // clang-format on
7280
7281 setValue(&I, DAG.getNode(Opcode, sdl,
7282 getValue(I.getArgOperand(0)).getValueType(),
7283 getValue(I.getArgOperand(0)), Flags));
7284 return;
7285 }
7286 case Intrinsic::atan2:
7287 setValue(&I, DAG.getNode(ISD::FATAN2, sdl,
7288 getValue(I.getArgOperand(0)).getValueType(),
7289 getValue(I.getArgOperand(0)),
7290 getValue(I.getArgOperand(1)), Flags));
7291 return;
7292 case Intrinsic::lround:
7293 case Intrinsic::llround:
7294 case Intrinsic::lrint:
7295 case Intrinsic::llrint: {
7296 unsigned Opcode;
7297 // clang-format off
7298 switch (Intrinsic) {
7299 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7300 case Intrinsic::lround: Opcode = ISD::LROUND; break;
7301 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
7302 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
7303 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
7304 }
7305 // clang-format on
7306
7307 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7308 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
7309 getValue(I.getArgOperand(0))));
7310 return;
7311 }
7312 case Intrinsic::minnum:
7313 setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
7314 getValue(I.getArgOperand(0)).getValueType(),
7315 getValue(I.getArgOperand(0)),
7316 getValue(I.getArgOperand(1)), Flags));
7317 return;
7318 case Intrinsic::maxnum:
7319 setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
7320 getValue(I.getArgOperand(0)).getValueType(),
7321 getValue(I.getArgOperand(0)),
7322 getValue(I.getArgOperand(1)), Flags));
7323 return;
7324 case Intrinsic::minimum:
7325 setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
7326 getValue(I.getArgOperand(0)).getValueType(),
7327 getValue(I.getArgOperand(0)),
7328 getValue(I.getArgOperand(1)), Flags));
7329 return;
7330 case Intrinsic::maximum:
7331 setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
7332 getValue(I.getArgOperand(0)).getValueType(),
7333 getValue(I.getArgOperand(0)),
7334 getValue(I.getArgOperand(1)), Flags));
7335 return;
7336 case Intrinsic::minimumnum:
7337 setValue(&I, DAG.getNode(ISD::FMINIMUMNUM, sdl,
7338 getValue(I.getArgOperand(0)).getValueType(),
7339 getValue(I.getArgOperand(0)),
7340 getValue(I.getArgOperand(1)), Flags));
7341 return;
7342 case Intrinsic::maximumnum:
7343 setValue(&I, DAG.getNode(ISD::FMAXIMUMNUM, sdl,
7344 getValue(I.getArgOperand(0)).getValueType(),
7345 getValue(I.getArgOperand(0)),
7346 getValue(I.getArgOperand(1)), Flags));
7347 return;
7348 case Intrinsic::copysign:
7349 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
7350 getValue(I.getArgOperand(0)).getValueType(),
7351 getValue(I.getArgOperand(0)),
7352 getValue(I.getArgOperand(1)), Flags));
7353 return;
7354 case Intrinsic::ldexp:
7355 setValue(&I, DAG.getNode(ISD::FLDEXP, sdl,
7356 getValue(I.getArgOperand(0)).getValueType(),
7357 getValue(I.getArgOperand(0)),
7358 getValue(I.getArgOperand(1)), Flags));
7359 return;
7360 case Intrinsic::modf:
7361 case Intrinsic::sincos:
7362 case Intrinsic::sincospi:
7363 case Intrinsic::frexp: {
7364 unsigned Opcode;
7365 switch (Intrinsic) {
7366 default:
7367 llvm_unreachable("unexpected intrinsic");
7368 case Intrinsic::sincos:
7369 Opcode = ISD::FSINCOS;
7370 break;
7371 case Intrinsic::sincospi:
7372 Opcode = ISD::FSINCOSPI;
7373 break;
7374 case Intrinsic::modf:
7375 Opcode = ISD::FMODF;
7376 break;
7377 case Intrinsic::frexp:
7378 Opcode = ISD::FFREXP;
7379 break;
7380 }
7381 SmallVector<EVT, 2> ValueVTs;
7382 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
7383 SDVTList VTs = DAG.getVTList(ValueVTs);
7384 setValue(
7385 &I, DAG.getNode(Opcode, sdl, VTs, getValue(I.getArgOperand(0)), Flags));
7386 return;
7387 }
7388 case Intrinsic::arithmetic_fence: {
7389 setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
7390 getValue(I.getArgOperand(0)).getValueType(),
7391 getValue(I.getArgOperand(0)), Flags));
7392 return;
7393 }
7394 case Intrinsic::fma:
7395 setValue(&I, DAG.getNode(
7396 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
7397 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
7398 getValue(I.getArgOperand(2)), Flags));
7399 return;
7400#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7401 case Intrinsic::INTRINSIC:
7402#include "llvm/IR/ConstrainedOps.def"
7403 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
7404 return;
7405#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7406#include "llvm/IR/VPIntrinsics.def"
7407 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
7408 return;
7409 case Intrinsic::fptrunc_round: {
7410 // Get the last argument, the metadata and convert it to an integer in the
7411 // call
7412 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7413 std::optional<RoundingMode> RoundMode =
7414 convertStrToRoundingMode(cast<MDString>(MD)->getString());
7415
7416 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7417
7418 // Propagate fast-math-flags from IR to node(s).
7419 SDNodeFlags Flags;
7420 Flags.copyFMF(*cast<FPMathOperator>(&I));
7421 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
7422
7424 Result = DAG.getNode(
7425 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
7426 DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));
7427 setValue(&I, Result);
7428
7429 return;
7430 }
7431 case Intrinsic::fmuladd: {
7432 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7433 if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
7434 setValue(&I, DAG.getNode(ISD::FMA, sdl,
7435 getValue(I.getArgOperand(0)).getValueType(),
7436 getValue(I.getArgOperand(0)),
7437 getValue(I.getArgOperand(1)),
7438 getValue(I.getArgOperand(2)), Flags));
7439 } else if (TLI.isOperationLegalOrCustom(ISD::FMULADD, VT)) {
7440 // TODO: Support splitting the vector.
7441 setValue(&I, DAG.getNode(ISD::FMULADD, sdl,
7442 getValue(I.getArgOperand(0)).getValueType(),
7443 getValue(I.getArgOperand(0)),
7444 getValue(I.getArgOperand(1)),
7445 getValue(I.getArgOperand(2)), Flags));
7446 } else {
7447 // TODO: Intrinsic calls should have fast-math-flags.
7448 SDValue Mul = DAG.getNode(
7449 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
7450 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
7451 SDValue Add = DAG.getNode(ISD::FADD, sdl,
7452 getValue(I.getArgOperand(0)).getValueType(),
7453 Mul, getValue(I.getArgOperand(2)), Flags);
7454 setValue(&I, Add);
7455 }
7456 return;
7457 }
7458 case Intrinsic::fptosi_sat: {
7459 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7460 setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
7461 getValue(I.getArgOperand(0)),
7462 DAG.getValueType(VT.getScalarType())));
7463 return;
7464 }
7465 case Intrinsic::fptoui_sat: {
7466 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7467 setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
7468 getValue(I.getArgOperand(0)),
7469 DAG.getValueType(VT.getScalarType())));
7470 return;
7471 }
7472 case Intrinsic::convert_from_arbitrary_fp: {
7473 // Extract format metadata and convert to semantics enum.
7474 EVT DstVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7475 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7476 StringRef FormatStr = cast<MDString>(MD)->getString();
7477 const fltSemantics *SrcSem =
7479 if (!SrcSem) {
7480 DAG.getContext()->emitError(
7481 "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7482 "'");
7483 setValue(&I, DAG.getPOISON(DstVT));
7484 return;
7485 }
7487
7488 SDValue IntVal = getValue(I.getArgOperand(0));
7489
7490 // Emit ISD::CONVERT_FROM_ARBITRARY_FP node.
7491 SDValue SemConst =
7492 DAG.getTargetConstant(static_cast<int>(SemEnum), sdl, MVT::i32);
7493 setValue(&I, DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, sdl, DstVT, IntVal,
7494 SemConst));
7495 return;
7496 }
7497 case Intrinsic::convert_to_arbitrary_fp: {
7498 // Extract format metadata and convert to semantics enum.
7499 EVT DstVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7500 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7501 StringRef FormatStr = cast<MDString>(MD)->getString();
7502 const fltSemantics *DstSem =
7504 if (!DstSem) {
7505 DAG.getContext()->emitError(
7506 "convert_to_arbitrary_fp: not implemented format '" + FormatStr +
7507 "'");
7508 setValue(&I, DAG.getPOISON(DstVT));
7509 return;
7510 }
7512
7513 Metadata *RoundMD =
7514 cast<MetadataAsValue>(I.getArgOperand(2))->getMetadata();
7515 StringRef RoundStr = cast<MDString>(RoundMD)->getString();
7516 std::optional<RoundingMode> RoundMode = convertStrToRoundingMode(RoundStr);
7517 assert(RoundMode && *RoundMode != RoundingMode::Dynamic &&
7518 "Dynamic rounding mode should have been rejected by the verifier");
7519
7520 uint64_t Saturate =
7521 cast<ConstantInt>(I.getArgOperand(3))->getZExtValue() ? 1 : 0;
7522
7523 SDValue FloatVal = getValue(I.getArgOperand(0));
7524
7525 SDValue SemConst =
7526 DAG.getTargetConstant(static_cast<int>(SemEnum), sdl, MVT::i32);
7527 SDValue RoundConst =
7528 DAG.getTargetConstant(static_cast<int>(*RoundMode), sdl, MVT::i32);
7529 SDValue SatConst = DAG.getTargetConstant(Saturate, sdl, MVT::i32);
7530 setValue(&I, DAG.getNode(ISD::CONVERT_TO_ARBITRARY_FP, sdl, DstVT, FloatVal,
7531 SemConst, RoundConst, SatConst));
7532 return;
7533 }
7534 case Intrinsic::set_rounding:
7535 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
7536 {getRoot(), getValue(I.getArgOperand(0))});
7537 setValue(&I, Res);
7538 DAG.setRoot(Res.getValue(0));
7539 return;
7540 case Intrinsic::is_fpclass: {
7541 const DataLayout DLayout = DAG.getDataLayout();
7542 EVT DestVT = TLI.getValueType(DLayout, I.getType());
7543 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
7544 FPClassTest Test = static_cast<FPClassTest>(
7545 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
7546 MachineFunction &MF = DAG.getMachineFunction();
7547 const Function &F = MF.getFunction();
7548 SDValue Op = getValue(I.getArgOperand(0));
7549 SDNodeFlags Flags;
7550 Flags.setNoFPExcept(
7551 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7552 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7553 // expansion can use illegal types. Making expansion early allows
7554 // legalizing these types prior to selection.
7555 if (!TLI.isOperationLegal(ISD::IS_FPCLASS, ArgVT) &&
7557 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
7558 setValue(&I, Result);
7559 return;
7560 }
7561
7562 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
7563 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
7564 setValue(&I, V);
7565 return;
7566 }
7567 case Intrinsic::get_fpenv: {
7568 const DataLayout DLayout = DAG.getDataLayout();
7569 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
7570 Align TempAlign = DAG.getEVTAlign(EnvVT);
7571 SDValue Chain = getRoot();
7572 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7573 // and temporary storage in stack.
7575 Res = DAG.getNode(
7576 ISD::GET_FPENV, sdl,
7577 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7578 MVT::Other),
7579 Chain);
7580 } else {
7581 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7582 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7583 auto MPI =
7584 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
7585 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7587 TempAlign);
7588 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7589 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
7590 }
7591 setValue(&I, Res);
7592 DAG.setRoot(Res.getValue(1));
7593 return;
7594 }
7595 case Intrinsic::set_fpenv: {
7596 SDValue Env = getValue(I.getArgOperand(0));
7597 EVT EnvVT = Env.getValueType();
7598 Align TempAlign = DAG.getEVTAlign(EnvVT);
7599 SDValue Chain = getRoot();
7600 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7601 // environment from memory.
7603 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
7604 } else {
7605 // Allocate space in stack, copy environment bits into it and use this
7606 // memory in SET_FPENV_MEM.
7607 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7608 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7609 auto MPI =
7610 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
7611 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7613 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7615 TempAlign);
7616 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7617 }
7618 DAG.setRoot(Chain);
7619 return;
7620 }
7621 case Intrinsic::reset_fpenv:
7622 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
7623 return;
7624 case Intrinsic::get_fpmode:
7625 Res = DAG.getNode(
7626 ISD::GET_FPMODE, sdl,
7627 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7628 MVT::Other),
7629 DAG.getRoot());
7630 setValue(&I, Res);
7631 DAG.setRoot(Res.getValue(1));
7632 return;
7633 case Intrinsic::set_fpmode:
7634 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
7635 getValue(I.getArgOperand(0)));
7636 DAG.setRoot(Res);
7637 return;
7638 case Intrinsic::reset_fpmode: {
7639 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
7640 DAG.setRoot(Res);
7641 return;
7642 }
7643 case Intrinsic::pcmarker: {
7644 SDValue Tmp = getValue(I.getArgOperand(0));
7645 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
7646 return;
7647 }
7648 case Intrinsic::readcyclecounter: {
7649 SDValue Op = getRoot();
7650 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
7651 DAG.getVTList(MVT::i64, MVT::Other), Op);
7652 setValue(&I, Res);
7653 DAG.setRoot(Res.getValue(1));
7654 return;
7655 }
7656 case Intrinsic::readsteadycounter: {
7657 SDValue Op = getRoot();
7658 Res = DAG.getNode(ISD::READSTEADYCOUNTER, sdl,
7659 DAG.getVTList(MVT::i64, MVT::Other), Op);
7660 setValue(&I, Res);
7661 DAG.setRoot(Res.getValue(1));
7662 return;
7663 }
7664 case Intrinsic::bitreverse:
7665 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
7666 getValue(I.getArgOperand(0)).getValueType(),
7667 getValue(I.getArgOperand(0))));
7668 return;
7669 case Intrinsic::bswap:
7670 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
7671 getValue(I.getArgOperand(0)).getValueType(),
7672 getValue(I.getArgOperand(0))));
7673 return;
7674 case Intrinsic::cttz: {
7675 SDValue Arg = getValue(I.getArgOperand(0));
7676 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7677 EVT Ty = Arg.getValueType();
7678 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_POISON,
7679 sdl, Ty, Arg));
7680 return;
7681 }
7682 case Intrinsic::ctlz: {
7683 SDValue Arg = getValue(I.getArgOperand(0));
7684 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7685 EVT Ty = Arg.getValueType();
7686 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_POISON,
7687 sdl, Ty, Arg));
7688 return;
7689 }
7690 case Intrinsic::ctpop: {
7691 SDValue Arg = getValue(I.getArgOperand(0));
7692 EVT Ty = Arg.getValueType();
7693 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7694 return;
7695 }
7696 case Intrinsic::fshl:
7697 case Intrinsic::fshr: {
7698 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7699 SDValue X = getValue(I.getArgOperand(0));
7700 SDValue Y = getValue(I.getArgOperand(1));
7701 SDValue Z = getValue(I.getArgOperand(2));
7702 EVT VT = X.getValueType();
7703
7704 if (X == Y) {
7705 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7706 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7707 } else {
7708 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7709 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7710 }
7711 return;
7712 }
7713 case Intrinsic::clmul: {
7714 SDValue X = getValue(I.getArgOperand(0));
7715 SDValue Y = getValue(I.getArgOperand(1));
7716 setValue(&I, DAG.getNode(ISD::CLMUL, sdl, X.getValueType(), X, Y));
7717 return;
7718 }
7719 case Intrinsic::pext: {
7720 SDValue X = getValue(I.getArgOperand(0));
7721 SDValue Y = getValue(I.getArgOperand(1));
7722 setValue(&I, DAG.getNode(ISD::PEXT, sdl, X.getValueType(), X, Y));
7723 return;
7724 }
7725 case Intrinsic::pdep: {
7726 SDValue X = getValue(I.getArgOperand(0));
7727 SDValue Y = getValue(I.getArgOperand(1));
7728 setValue(&I, DAG.getNode(ISD::PDEP, sdl, X.getValueType(), X, Y));
7729 return;
7730 }
7731 case Intrinsic::smulh:
7732 case Intrinsic::umulh: {
7733 auto Opc = Intrinsic == Intrinsic::smulh ? ISD::MULHS : ISD::MULHU;
7734 SDValue X = getValue(I.getArgOperand(0));
7735 SDValue Y = getValue(I.getArgOperand(1));
7736 setValue(&I, DAG.getNode(Opc, sdl, X.getValueType(), X, Y));
7737 return;
7738 }
7739 case Intrinsic::sadd_sat: {
7740 SDValue Op1 = getValue(I.getArgOperand(0));
7741 SDValue Op2 = getValue(I.getArgOperand(1));
7742 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7743 return;
7744 }
7745 case Intrinsic::uadd_sat: {
7746 SDValue Op1 = getValue(I.getArgOperand(0));
7747 SDValue Op2 = getValue(I.getArgOperand(1));
7748 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7749 return;
7750 }
7751 case Intrinsic::ssub_sat: {
7752 SDValue Op1 = getValue(I.getArgOperand(0));
7753 SDValue Op2 = getValue(I.getArgOperand(1));
7754 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7755 return;
7756 }
7757 case Intrinsic::usub_sat: {
7758 SDValue Op1 = getValue(I.getArgOperand(0));
7759 SDValue Op2 = getValue(I.getArgOperand(1));
7760 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7761 return;
7762 }
7763 case Intrinsic::sshl_sat:
7764 case Intrinsic::ushl_sat: {
7765 SDValue Op1 = getValue(I.getArgOperand(0));
7766 SDValue Op2 = getValue(I.getArgOperand(1));
7767
7768 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
7769 Op1.getValueType(), DAG.getDataLayout());
7770
7771 // Coerce the shift amount to the right type if we can. This exposes the
7772 // truncate or zext to optimization early.
7773 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
7774 assert(ShiftTy.getSizeInBits() >=
7776 "Unexpected shift type");
7777 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
7778 }
7779
7780 unsigned Opc =
7781 Intrinsic == Intrinsic::sshl_sat ? ISD::SSHLSAT : ISD::USHLSAT;
7782 setValue(&I, DAG.getNode(Opc, sdl, Op1.getValueType(), Op1, Op2));
7783 return;
7784 }
7785 case Intrinsic::smul_fix:
7786 case Intrinsic::umul_fix:
7787 case Intrinsic::smul_fix_sat:
7788 case Intrinsic::umul_fix_sat: {
7789 SDValue Op1 = getValue(I.getArgOperand(0));
7790 SDValue Op2 = getValue(I.getArgOperand(1));
7791 SDValue Op3 = getValue(I.getArgOperand(2));
7792 setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
7793 Op1.getValueType(), Op1, Op2, Op3));
7794 return;
7795 }
7796 case Intrinsic::sdiv_fix:
7797 case Intrinsic::udiv_fix:
7798 case Intrinsic::sdiv_fix_sat:
7799 case Intrinsic::udiv_fix_sat: {
7800 SDValue Op1 = getValue(I.getArgOperand(0));
7801 SDValue Op2 = getValue(I.getArgOperand(1));
7802 SDValue Op3 = getValue(I.getArgOperand(2));
7804 Op1, Op2, Op3, DAG, TLI));
7805 return;
7806 }
7807 case Intrinsic::smax: {
7808 SDValue Op1 = getValue(I.getArgOperand(0));
7809 SDValue Op2 = getValue(I.getArgOperand(1));
7810 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7811 return;
7812 }
7813 case Intrinsic::smin: {
7814 SDValue Op1 = getValue(I.getArgOperand(0));
7815 SDValue Op2 = getValue(I.getArgOperand(1));
7816 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7817 return;
7818 }
7819 case Intrinsic::umax: {
7820 SDValue Op1 = getValue(I.getArgOperand(0));
7821 SDValue Op2 = getValue(I.getArgOperand(1));
7822 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7823 return;
7824 }
7825 case Intrinsic::umin: {
7826 SDValue Op1 = getValue(I.getArgOperand(0));
7827 SDValue Op2 = getValue(I.getArgOperand(1));
7828 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7829 return;
7830 }
7831 case Intrinsic::abs: {
7832 SDValue Op1 = getValue(I.getArgOperand(0));
7833 bool IntMinIsPoison = cast<ConstantInt>(I.getArgOperand(1))->isOne();
7834 unsigned Opc = IntMinIsPoison ? ISD::ABS_MIN_POISON : ISD::ABS;
7835 setValue(&I, DAG.getNode(Opc, sdl, Op1.getValueType(), Op1));
7836 return;
7837 }
7838 case Intrinsic::scmp: {
7839 SDValue Op1 = getValue(I.getArgOperand(0));
7840 SDValue Op2 = getValue(I.getArgOperand(1));
7841 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7842 setValue(&I, DAG.getNode(ISD::SCMP, sdl, DestVT, Op1, Op2));
7843 break;
7844 }
7845 case Intrinsic::ucmp: {
7846 SDValue Op1 = getValue(I.getArgOperand(0));
7847 SDValue Op2 = getValue(I.getArgOperand(1));
7848 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7849 setValue(&I, DAG.getNode(ISD::UCMP, sdl, DestVT, Op1, Op2));
7850 break;
7851 }
7852 case Intrinsic::stackaddress:
7853 case Intrinsic::stacksave: {
7854 unsigned SDOpcode = Intrinsic == Intrinsic::stackaddress ? ISD::STACKADDRESS
7856 SDValue Op = getRoot();
7857 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7858 Res = DAG.getNode(SDOpcode, sdl, DAG.getVTList(VT, MVT::Other), Op);
7859 setValue(&I, Res);
7860 DAG.setRoot(Res.getValue(1));
7861 return;
7862 }
7863 case Intrinsic::stackrestore:
7864 Res = getValue(I.getArgOperand(0));
7865 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7866 return;
7867 case Intrinsic::get_dynamic_area_offset: {
7868 SDValue Op = getRoot();
7869 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7870 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
7871 Op);
7872 DAG.setRoot(Op);
7873 setValue(&I, Res);
7874 return;
7875 }
7876 case Intrinsic::stackguard: {
7877 MachineFunction &MF = DAG.getMachineFunction();
7878 const Module &M = *MF.getFunction().getParent();
7879 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7880 SDValue Chain = getRoot();
7881 if (TLI.useLoadStackGuardNode(M)) {
7882 Res = getLoadStackGuard(DAG, sdl, Chain);
7883 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7884 } else {
7885 const Value *Global = TLI.getSDagStackGuard(M, DAG.getLibcalls());
7886 if (!Global) {
7887 LLVMContext &Ctx = *DAG.getContext();
7888 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
7889 setValue(&I, DAG.getPOISON(PtrTy));
7890 return;
7891 }
7892
7893 Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
7894 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7895 MachinePointerInfo(Global, 0), Align,
7897 }
7898 // Mix the cookie with FP if enabled. Skip if using LOAD_STACK_GUARD
7899 // with post-RA mixing (AArch64 MSVCRT), as the mixing will be done during
7900 // post-RA expansion of LOAD_STACK_GUARD.
7902 Res = TLI.emitStackGuardMixFP(DAG, Res, sdl);
7903 DAG.setRoot(Chain);
7904 setValue(&I, Res);
7905 return;
7906 }
7907 case Intrinsic::stackprotector: {
7908 // Emit code into the DAG to store the stack guard onto the stack.
7909 MachineFunction &MF = DAG.getMachineFunction();
7910 MachineFrameInfo &MFI = MF.getFrameInfo();
7911 const Module &M = *MF.getFunction().getParent();
7912 SDValue Src, Chain = getRoot();
7913
7915 Src = getLoadStackGuard(DAG, sdl, Chain);
7916 else
7917 Src = getValue(I.getArgOperand(0)); // The guard's value.
7918
7919 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7920
7921 int FI = FuncInfo.StaticAllocaMap[Slot];
7922 MFI.setStackProtectorIndex(FI);
7923 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7924
7925 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7926
7927 // Store the stack protector onto the stack.
7928 Res = DAG.getStore(
7929 Chain, sdl, Src, FIN,
7930 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
7931 MaybeAlign(), MachineMemOperand::MOVolatile);
7932 setValue(&I, Res);
7933 DAG.setRoot(Res);
7934 return;
7935 }
7936 case Intrinsic::objectsize:
7937 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7938
7939 case Intrinsic::is_constant:
7940 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7941
7942 case Intrinsic::annotation:
7943 case Intrinsic::ptr_annotation:
7944 case Intrinsic::launder_invariant_group:
7945 // Drop the intrinsic, but forward the value
7946 setValue(&I, getValue(I.getOperand(0)));
7947 return;
7948
7949 case Intrinsic::type_test:
7950 case Intrinsic::public_type_test:
7951 case Intrinsic::type_checked_load:
7952 case Intrinsic::type_checked_load_relative: {
7953 // These intrinsics are expected to be lowered by the LowerTypeTests pass
7954 // before code generation. Surviving until here usually indicates a
7955 // misconfiguration, for instance when devirtualization is enabled but LTO
7956 // does not actually run.
7957 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
7958 *I.getFunction(),
7959 Intrinsic::getBaseName(Intrinsic) +
7960 " intrinsic must be lowered by the LowerTypeTests pass "
7961 "before code generation",
7962 sdl.getDebugLoc()));
7963
7964 // Lower the result to poison so that compilation can continue and collect
7965 // any further diagnostics.
7966 setValueToPoison(&I, sdl);
7967 return;
7968 }
7969
7970 case Intrinsic::assume:
7971 case Intrinsic::experimental_noalias_scope_decl:
7972 case Intrinsic::var_annotation:
7973 case Intrinsic::sideeffect:
7974 // Discard annotate attributes, noalias scope declarations, assumptions, and
7975 // artificial side-effects.
7976 return;
7977
7978 case Intrinsic::codeview_annotation: {
7979 // Emit a label associated with this metadata.
7980 MachineFunction &MF = DAG.getMachineFunction();
7981 MCSymbol *Label = MF.getContext().createTempSymbol("annotation", true);
7982 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7983 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7984 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7985 DAG.setRoot(Res);
7986 return;
7987 }
7988
7989 case Intrinsic::init_trampoline: {
7990 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7991
7992 SDValue Ops[6];
7993 Ops[0] = getRoot();
7994 Ops[1] = getValue(I.getArgOperand(0));
7995 Ops[2] = getValue(I.getArgOperand(1));
7996 Ops[3] = getValue(I.getArgOperand(2));
7997 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7998 Ops[5] = DAG.getSrcValue(F);
7999
8000 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
8001
8002 DAG.setRoot(Res);
8003 return;
8004 }
8005 case Intrinsic::adjust_trampoline:
8006 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
8007 TLI.getPointerTy(DAG.getDataLayout()),
8008 getValue(I.getArgOperand(0))));
8009 return;
8010 case Intrinsic::gcroot: {
8011 assert(DAG.getMachineFunction().getFunction().hasGC() &&
8012 "only valid in functions with gc specified, enforced by Verifier");
8013 assert(GFI && "implied by previous");
8014 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
8015 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
8016
8017 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
8018 GFI->addStackRoot(FI->getIndex(), TypeMap);
8019 return;
8020 }
8021 case Intrinsic::gcread:
8022 case Intrinsic::gcwrite:
8023 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
8024 case Intrinsic::get_rounding:
8025 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
8026 setValue(&I, Res);
8027 DAG.setRoot(Res.getValue(1));
8028 return;
8029
8030 case Intrinsic::expect:
8031 case Intrinsic::expect_with_probability:
8032 // Just replace __builtin_expect(exp, c) and
8033 // __builtin_expect_with_probability(exp, c, p) with EXP.
8034 setValue(&I, getValue(I.getArgOperand(0)));
8035 return;
8036
8037 case Intrinsic::ubsantrap:
8038 case Intrinsic::debugtrap:
8039 case Intrinsic::trap: {
8040 StringRef TrapFuncName =
8041 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
8042 if (TrapFuncName.empty()) {
8043 switch (Intrinsic) {
8044 case Intrinsic::trap:
8045 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
8046 break;
8047 case Intrinsic::debugtrap:
8048 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
8049 break;
8050 case Intrinsic::ubsantrap:
8051 DAG.setRoot(DAG.getNode(
8052 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
8053 DAG.getTargetConstant(
8054 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
8055 MVT::i32)));
8056 break;
8057 default: llvm_unreachable("unknown trap intrinsic");
8058 }
8059 DAG.addNoMergeSiteInfo(DAG.getRoot().getNode(),
8060 I.hasFnAttr(Attribute::NoMerge));
8061 return;
8062 }
8064 if (Intrinsic == Intrinsic::ubsantrap) {
8065 Value *Arg = I.getArgOperand(0);
8066 Args.emplace_back(Arg, getValue(Arg),
8067 LibInfo->getExtAttrForI8Param(/*Signed=*/false));
8068 }
8069
8070 TargetLowering::CallLoweringInfo CLI(DAG);
8071 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
8072 CallingConv::C, I.getType(),
8073 DAG.getExternalSymbol(TrapFuncName.data(),
8074 TLI.getPointerTy(DAG.getDataLayout())),
8075 std::move(Args));
8076 CLI.NoMerge = I.hasFnAttr(Attribute::NoMerge);
8077 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8078 DAG.setRoot(Result.second);
8079 return;
8080 }
8081
8082 case Intrinsic::allow_runtime_check:
8083 case Intrinsic::allow_ubsan_check:
8084 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
8085 return;
8086
8087 case Intrinsic::uadd_with_overflow:
8088 case Intrinsic::sadd_with_overflow:
8089 case Intrinsic::usub_with_overflow:
8090 case Intrinsic::ssub_with_overflow:
8091 case Intrinsic::umul_with_overflow:
8092 case Intrinsic::smul_with_overflow: {
8094 switch (Intrinsic) {
8095 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8096 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
8097 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
8098 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
8099 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
8100 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
8101 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
8102 }
8103 SDValue Op1 = getValue(I.getArgOperand(0));
8104 SDValue Op2 = getValue(I.getArgOperand(1));
8105
8106 EVT ResultVT = Op1.getValueType();
8107 EVT OverflowVT = ResultVT.changeElementType(*Context, MVT::i1);
8108
8109 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
8110 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
8111 return;
8112 }
8113 case Intrinsic::prefetch: {
8114 SDValue Ops[5];
8115 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
8117 Ops[0] = DAG.getRoot();
8118 Ops[1] = getValue(I.getArgOperand(0));
8119 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
8120 MVT::i32);
8121 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
8122 MVT::i32);
8123 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
8124 MVT::i32);
8125 SDValue Result = DAG.getMemIntrinsicNode(
8126 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
8127 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
8128 /* align */ std::nullopt, Flags);
8129
8130 // Chain the prefetch in parallel with any pending loads, to stay out of
8131 // the way of later optimizations.
8132 PendingLoads.push_back(Result);
8133 Result = getRoot();
8134 DAG.setRoot(Result);
8135 return;
8136 }
8137 case Intrinsic::lifetime_start:
8138 case Intrinsic::lifetime_end: {
8139 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
8140 // Stack coloring is not enabled in O0, discard region information.
8141 if (TM.getOptLevel() == CodeGenOptLevel::None)
8142 return;
8143
8144 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(I.getArgOperand(0));
8145 if (!LifetimeObject)
8146 return;
8147
8148 // First check that the Alloca is static, otherwise it won't have a
8149 // valid frame index.
8150 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
8151 if (SI == FuncInfo.StaticAllocaMap.end())
8152 return;
8153
8154 const int FrameIndex = SI->second;
8155 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex);
8156 DAG.setRoot(Res);
8157 return;
8158 }
8159 case Intrinsic::pseudoprobe: {
8160 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
8161 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
8162 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
8163 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
8164 DAG.setRoot(Res);
8165 return;
8166 }
8167 case Intrinsic::invariant_start:
8168 // Discard region information.
8169 setValue(&I,
8170 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
8171 return;
8172 case Intrinsic::invariant_end:
8173 // Discard region information.
8174 return;
8175 case Intrinsic::clear_cache: {
8176 SDValue InputChain = DAG.getRoot();
8177 SDValue StartVal = getValue(I.getArgOperand(0));
8178 SDValue EndVal = getValue(I.getArgOperand(1));
8179 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),
8180 {InputChain, StartVal, EndVal});
8181 setValue(&I, Res);
8182 DAG.setRoot(Res);
8183 return;
8184 }
8185 case Intrinsic::donothing:
8186 case Intrinsic::seh_try_begin:
8187 case Intrinsic::seh_scope_begin:
8188 case Intrinsic::seh_try_end:
8189 case Intrinsic::seh_scope_end:
8190 // ignore
8191 return;
8192 case Intrinsic::experimental_stackmap:
8193 visitStackmap(I);
8194 return;
8195 case Intrinsic::experimental_patchpoint_void:
8196 case Intrinsic::experimental_patchpoint:
8197 visitPatchpoint(I);
8198 return;
8199 case Intrinsic::experimental_gc_statepoint:
8201 return;
8202 case Intrinsic::experimental_gc_result:
8203 visitGCResult(cast<GCResultInst>(I));
8204 return;
8205 case Intrinsic::experimental_gc_relocate:
8206 visitGCRelocate(cast<GCRelocateInst>(I));
8207 return;
8208 case Intrinsic::instrprof_cover:
8209 llvm_unreachable("instrprof failed to lower a cover");
8210 case Intrinsic::instrprof_increment:
8211 llvm_unreachable("instrprof failed to lower an increment");
8212 case Intrinsic::instrprof_timestamp:
8213 llvm_unreachable("instrprof failed to lower a timestamp");
8214 case Intrinsic::instrprof_value_profile:
8215 llvm_unreachable("instrprof failed to lower a value profiling call");
8216 case Intrinsic::instrprof_mcdc_parameters:
8217 llvm_unreachable("instrprof failed to lower mcdc parameters");
8218 case Intrinsic::instrprof_mcdc_tvbitmap_update:
8219 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
8220 case Intrinsic::localescape: {
8221 MachineFunction &MF = DAG.getMachineFunction();
8222 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
8223
8224 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
8225 // is the same on all targets.
8226 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
8227 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
8228 if (isa<ConstantPointerNull>(Arg))
8229 continue; // Skip null pointers. They represent a hole in index space.
8230 AllocaInst *Slot = cast<AllocaInst>(Arg);
8231 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
8232 "can only escape static allocas");
8233 int FI = FuncInfo.StaticAllocaMap[Slot];
8234 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8236 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
8237 TII->get(TargetOpcode::LOCAL_ESCAPE))
8238 .addSym(FrameAllocSym)
8239 .addFrameIndex(FI);
8240 }
8241
8242 return;
8243 }
8244
8245 case Intrinsic::localrecover: {
8246 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
8247 MachineFunction &MF = DAG.getMachineFunction();
8248
8249 // Get the symbol that defines the frame offset.
8250 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
8251 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
8252 unsigned IdxVal =
8253 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
8254 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8256
8257 Value *FP = I.getArgOperand(1);
8258 SDValue FPVal = getValue(FP);
8259 EVT PtrVT = FPVal.getValueType();
8260
8261 // Create a MCSymbol for the label to avoid any target lowering
8262 // that would make this PC relative.
8263 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
8264 SDValue OffsetVal =
8265 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
8266
8267 // Add the offset to the FP.
8268 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
8269 setValue(&I, Add);
8270
8271 return;
8272 }
8273
8274 case Intrinsic::fake_use: {
8275 Value *V = I.getArgOperand(0);
8276 SDValue Ops[2];
8277 // For Values not declared or previously used in this basic block, the
8278 // NodeMap will not have an entry, and `getValue` will assert if V has no
8279 // valid register value.
8280 auto FakeUseValue = [&]() -> SDValue {
8281 SDValue &N = NodeMap[V];
8282 if (N.getNode())
8283 return N;
8284
8285 // If there's a virtual register allocated and initialized for this
8286 // value, use it.
8287 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
8288 return copyFromReg;
8289 // FIXME: Do we want to preserve constants? It seems pointless.
8290 if (isa<Constant>(V))
8291 return getValue(V);
8292 return SDValue();
8293 }();
8294 if (!FakeUseValue || FakeUseValue.isUndef())
8295 return;
8296 Ops[0] = getRoot();
8297 Ops[1] = FakeUseValue;
8298 // Also, do not translate a fake use with an undef operand, or any other
8299 // empty SDValues.
8300 if (!Ops[1] || Ops[1].isUndef())
8301 return;
8302 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other, Ops));
8303 return;
8304 }
8305
8306 case Intrinsic::reloc_none: {
8307 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
8308 StringRef SymbolName = cast<MDString>(MD)->getString();
8309 SDValue Ops[2] = {
8310 getRoot(),
8311 DAG.getTargetExternalSymbol(
8312 SymbolName.data(), TLI.getProgramPointerTy(DAG.getDataLayout()))};
8313 DAG.setRoot(DAG.getNode(ISD::RELOC_NONE, sdl, MVT::Other, Ops));
8314 return;
8315 }
8316
8317 case Intrinsic::cond_loop: {
8318 SDValue InputChain = DAG.getRoot();
8319 SDValue P = getValue(I.getArgOperand(0));
8320 Res = DAG.getNode(ISD::COND_LOOP, sdl, DAG.getVTList(MVT::Other),
8321 {InputChain, P});
8322 setValue(&I, Res);
8323 DAG.setRoot(Res);
8324 return;
8325 }
8326
8327 case Intrinsic::eh_exceptionpointer:
8328 case Intrinsic::eh_exceptioncode: {
8329 // Get the exception pointer vreg, copy from it, and resize it to fit.
8330 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
8331 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
8332 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
8333 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
8334 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
8335 if (Intrinsic == Intrinsic::eh_exceptioncode)
8336 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
8337 setValue(&I, N);
8338 return;
8339 }
8340 case Intrinsic::xray_customevent: {
8341 // Here we want to make sure that the intrinsic behaves as if it has a
8342 // specific calling convention.
8343 const auto &Triple = DAG.getTarget().getTargetTriple();
8344 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64 &&
8345 Triple.getArch() != Triple::hexagon)
8346 return;
8347
8349
8350 // We want to say that we always want the arguments in registers.
8351 SDValue LogEntryVal = getValue(I.getArgOperand(0));
8352 SDValue StrSizeVal = getValue(I.getArgOperand(1));
8353 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8354 SDValue Chain = getRoot();
8355 Ops.push_back(LogEntryVal);
8356 Ops.push_back(StrSizeVal);
8357 Ops.push_back(Chain);
8358
8359 // We need to enforce the calling convention for the callsite, so that
8360 // argument ordering is enforced correctly, and that register allocation can
8361 // see that some registers may be assumed clobbered and have to preserve
8362 // them across calls to the intrinsic.
8363 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
8364 sdl, NodeTys, Ops);
8365 SDValue patchableNode = SDValue(MN, 0);
8366 DAG.setRoot(patchableNode);
8367 setValue(&I, patchableNode);
8368 return;
8369 }
8370 case Intrinsic::xray_typedevent: {
8371 // Here we want to make sure that the intrinsic behaves as if it has a
8372 // specific calling convention.
8373 const auto &Triple = DAG.getTarget().getTargetTriple();
8374 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64 &&
8375 Triple.getArch() != Triple::hexagon)
8376 return;
8377
8379
8380 // We want to say that we always want the arguments in registers.
8381 // It's unclear to me how manipulating the selection DAG here forces callers
8382 // to provide arguments in registers instead of on the stack.
8383 SDValue LogTypeId = getValue(I.getArgOperand(0));
8384 SDValue LogEntryVal = getValue(I.getArgOperand(1));
8385 SDValue StrSizeVal = getValue(I.getArgOperand(2));
8386 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8387 SDValue Chain = getRoot();
8388 Ops.push_back(LogTypeId);
8389 Ops.push_back(LogEntryVal);
8390 Ops.push_back(StrSizeVal);
8391 Ops.push_back(Chain);
8392
8393 // We need to enforce the calling convention for the callsite, so that
8394 // argument ordering is enforced correctly, and that register allocation can
8395 // see that some registers may be assumed clobbered and have to preserve
8396 // them across calls to the intrinsic.
8397 MachineSDNode *MN = DAG.getMachineNode(
8398 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
8399 SDValue patchableNode = SDValue(MN, 0);
8400 DAG.setRoot(patchableNode);
8401 setValue(&I, patchableNode);
8402 return;
8403 }
8404 case Intrinsic::experimental_deoptimize:
8406 return;
8407 case Intrinsic::stepvector:
8408 visitStepVector(I);
8409 return;
8410 case Intrinsic::vector_reduce_fadd:
8411 case Intrinsic::vector_reduce_fmul:
8412 case Intrinsic::vector_reduce_add:
8413 case Intrinsic::vector_reduce_mul:
8414 case Intrinsic::vector_reduce_and:
8415 case Intrinsic::vector_reduce_or:
8416 case Intrinsic::vector_reduce_xor:
8417 case Intrinsic::vector_reduce_smax:
8418 case Intrinsic::vector_reduce_smin:
8419 case Intrinsic::vector_reduce_umax:
8420 case Intrinsic::vector_reduce_umin:
8421 case Intrinsic::vector_reduce_fmax:
8422 case Intrinsic::vector_reduce_fmin:
8423 case Intrinsic::vector_reduce_fmaximum:
8424 case Intrinsic::vector_reduce_fminimum:
8425 case Intrinsic::vector_reduce_fmaximumnum:
8426 case Intrinsic::vector_reduce_fminimumnum:
8427 visitVectorReduce(I, Intrinsic);
8428 return;
8429
8430 case Intrinsic::icall_branch_funnel: {
8432 Ops.push_back(getValue(I.getArgOperand(0)));
8433
8434 int64_t Offset;
8436 I.getArgOperand(1), Offset, DAG.getDataLayout()));
8437 if (!Base)
8439 "llvm.icall.branch.funnel operand must be a GlobalValue");
8440 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
8441
8442 struct BranchFunnelTarget {
8443 int64_t Offset;
8445 };
8447
8448 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
8450 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
8451 if (ElemBase != Base)
8452 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
8453 "to the same GlobalValue");
8454
8455 SDValue Val = getValue(I.getArgOperand(Op + 1));
8456 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
8457 if (!GA)
8459 "llvm.icall.branch.funnel operand must be a GlobalValue");
8460 Targets.push_back({Offset, DAG.getTargetGlobalAddress(
8461 GA->getGlobal(), sdl, Val.getValueType(),
8462 GA->getOffset())});
8463 }
8464 llvm::sort(Targets,
8465 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
8466 return T1.Offset < T2.Offset;
8467 });
8468
8469 for (auto &T : Targets) {
8470 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
8471 Ops.push_back(T.Target);
8472 }
8473
8474 Ops.push_back(DAG.getRoot()); // Chain
8475 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
8476 MVT::Other, Ops),
8477 0);
8478 DAG.setRoot(N);
8479 setValue(&I, N);
8480 HasTailCall = true;
8481 return;
8482 }
8483
8484 case Intrinsic::wasm_landingpad_index:
8485 // Information this intrinsic contained has been transferred to
8486 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
8487 // delete it now.
8488 return;
8489
8490 case Intrinsic::aarch64_settag:
8491 case Intrinsic::aarch64_settag_zero: {
8492 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8493 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
8495 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
8496 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
8497 ZeroMemory);
8498 DAG.setRoot(Val);
8499 setValue(&I, Val);
8500 return;
8501 }
8502 case Intrinsic::amdgcn_cs_chain: {
8503 // At this point we don't care if it's amdgpu_cs_chain or
8504 // amdgpu_cs_chain_preserve.
8506
8507 Type *RetTy = I.getType();
8508 assert(RetTy->isVoidTy() && "Should not return");
8509
8510 SDValue Callee = getValue(I.getOperand(0));
8511
8512 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
8513 // We'll also tack the value of the EXEC mask at the end.
8515 Args.reserve(3);
8516
8517 for (unsigned Idx : {2, 3, 1}) {
8518 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
8519 I.getOperand(Idx)->getType());
8520 Arg.setAttributes(&I, Idx);
8521 Args.push_back(Arg);
8522 }
8523
8524 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
8525 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
8526 Args[2].IsInReg = true; // EXEC should be inreg
8527
8528 // Forward the flags and any additional arguments.
8529 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
8530 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
8531 I.getOperand(Idx)->getType());
8532 Arg.setAttributes(&I, Idx);
8533 Args.push_back(Arg);
8534 }
8535
8536 TargetLowering::CallLoweringInfo CLI(DAG);
8537 CLI.setDebugLoc(getCurSDLoc())
8538 .setChain(getRoot())
8539 .setCallee(CC, RetTy, Callee, std::move(Args))
8540 .setNoReturn(true)
8541 .setTailCall(true)
8542 .setConvergent(I.isConvergent());
8543 CLI.CB = &I;
8544 std::pair<SDValue, SDValue> Result =
8545 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
8546 (void)Result;
8547 assert(!Result.first.getNode() && !Result.second.getNode() &&
8548 "Should've lowered as tail call");
8549
8550 HasTailCall = true;
8551 return;
8552 }
8553 case Intrinsic::amdgcn_call_whole_wave: {
8555 bool isTailCall = I.isTailCall();
8556
8557 // The first argument is the callee. Skip it when assembling the call args.
8558 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
8559 TargetLowering::ArgListEntry Arg(getValue(I.getArgOperand(Idx)),
8560 I.getArgOperand(Idx)->getType());
8561 Arg.setAttributes(&I, Idx);
8562
8563 // If we have an explicit sret argument that is an Instruction, (i.e., it
8564 // might point to function-local memory), we can't meaningfully tail-call.
8565 if (Arg.IsSRet && isa<Instruction>(I.getArgOperand(Idx)))
8566 isTailCall = false;
8567
8568 Args.push_back(Arg);
8569 }
8570
8571 SDValue ConvControlToken;
8572 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8573 auto *Token = Bundle->Inputs[0].get();
8574 ConvControlToken = getValue(Token);
8575 }
8576
8577 TargetLowering::CallLoweringInfo CLI(DAG);
8578 CLI.setDebugLoc(getCurSDLoc())
8579 .setChain(getRoot())
8580 .setCallee(CallingConv::AMDGPU_Gfx_WholeWave, I.getType(),
8581 getValue(I.getArgOperand(0)), std::move(Args))
8582 .setTailCall(isTailCall && canTailCall(I))
8583 .setIsPreallocated(
8584 I.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
8585 .setConvergent(I.isConvergent())
8586 .setConvergenceControlToken(ConvControlToken);
8587 CLI.CB = &I;
8588
8589 std::pair<SDValue, SDValue> Result =
8590 lowerInvokable(CLI, /*EHPadBB=*/nullptr);
8591
8592 if (Result.first.getNode())
8593 setValue(&I, Result.first);
8594 return;
8595 }
8596 case Intrinsic::ptrmask: {
8597 SDValue Ptr = getValue(I.getOperand(0));
8598 SDValue Mask = getValue(I.getOperand(1));
8599
8600 // On arm64_32, pointers are 32 bits when stored in memory, but
8601 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
8602 // match the index type, but the pointer is 64 bits, so the mask must be
8603 // zero-extended up to 64 bits to match the pointer.
8604 EVT PtrVT =
8605 TLI.getValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8606 EVT MemVT =
8607 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8608 assert(PtrVT == Ptr.getValueType());
8609 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
8610 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
8611 // 128-bit, so we have to pad the mask with ones for unused bits.
8612 auto HighOnes = DAG.getNode(
8613 ISD::SHL, sdl, PtrVT, DAG.getAllOnesConstant(sdl, PtrVT),
8614 DAG.getShiftAmountConstant(Mask.getValueType().getFixedSizeInBits(),
8615 PtrVT, sdl));
8616 Mask = DAG.getNode(ISD::OR, sdl, PtrVT,
8617 DAG.getZExtOrTrunc(Mask, sdl, PtrVT), HighOnes);
8618 } else if (Mask.getValueType() != PtrVT)
8619 Mask = DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);
8620
8621 assert(Mask.getValueType() == PtrVT);
8622 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
8623 return;
8624 }
8625 case Intrinsic::threadlocal_address: {
8626 setValue(&I, getValue(I.getOperand(0)));
8627 return;
8628 }
8629 case Intrinsic::get_active_lane_mask: {
8630 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8631 SDValue Index = getValue(I.getOperand(0));
8632 SDValue TripCount = getValue(I.getOperand(1));
8633 EVT ElementVT = Index.getValueType();
8634
8635 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
8636 setValue(&I, DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, sdl, CCVT, Index,
8637 TripCount));
8638 return;
8639 }
8640
8641 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
8642 CCVT.getVectorElementCount());
8643
8644 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
8645 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
8646 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
8647 SDValue VectorInduction = DAG.getNode(
8648 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
8649 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
8650 VectorTripCount, ISD::CondCode::SETULT);
8651 setValue(&I, SetCC);
8652 return;
8653 }
8654 case Intrinsic::mask_beforefirst: {
8655 SDValue Op = getValue(I.getOperand(0));
8656 setValue(&I,
8657 DAG.getNode(ISD::MASK_BEFOREFIRST, sdl, Op.getValueType(), Op));
8658 return;
8659 }
8660 case Intrinsic::experimental_get_vector_length: {
8661 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8662 "Expected positive VF");
8663 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
8664 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
8665
8666 SDValue Count = getValue(I.getOperand(0));
8667 EVT CountVT = Count.getValueType();
8668
8669 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8670 visitTargetIntrinsic(I, Intrinsic);
8671 return;
8672 }
8673
8674 // Expand to a umin between the trip count and the maximum elements the type
8675 // can hold.
8676 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8677
8678 // Extend the trip count to at least the result VT.
8679 if (CountVT.bitsLT(VT)) {
8680 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
8681 CountVT = VT;
8682 }
8683
8684 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
8686
8687 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
8688 // Clip to the result type if needed.
8689 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
8690
8691 setValue(&I, Trunc);
8692 return;
8693 }
8694 case Intrinsic::vector_partial_reduce_add: {
8695 SDValue Acc = getValue(I.getOperand(0));
8696 SDValue Input = getValue(I.getOperand(1));
8697 setValue(&I,
8698 DAG.getNode(ISD::PARTIAL_REDUCE_UMLA, sdl, Acc.getValueType(), Acc,
8699 Input, DAG.getConstant(1, sdl, Input.getValueType())));
8700 return;
8701 }
8702 case Intrinsic::vector_partial_reduce_fadd: {
8703 SDNodeFlags Flags;
8704 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
8705 Flags.copyFMF(*FPOp);
8706 SDValue Acc = getValue(I.getOperand(0));
8707 SDValue Input = getValue(I.getOperand(1));
8708 setValue(&I, DAG.getNode(ISD::PARTIAL_REDUCE_FMLA, sdl, Acc.getValueType(),
8709 Acc, Input,
8710 DAG.getConstantFP(1.0, sdl, Input.getValueType()),
8711 Flags));
8712 return;
8713 }
8714 case Intrinsic::experimental_cttz_elts: {
8715 SDValue Op = getValue(I.getOperand(0));
8716 EVT OpVT = Op.getValueType();
8717 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
8718 bool ZeroIsPoison =
8719 !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
8720 if (OpVT.getVectorElementType() != MVT::i1) {
8721 // Compare the input vector elements to zero & use to count trailing
8722 // zeros.
8723 SDValue AllZero = DAG.getConstant(0, sdl, OpVT);
8724 EVT I1OpVT = OpVT.changeVectorElementType(*DAG.getContext(), MVT::i1);
8725 Op = DAG.getSetCC(sdl, I1OpVT, Op, AllZero, ISD::SETNE);
8726 }
8727 setValue(&I, DAG.getNode(ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
8729 sdl, RetTy, Op));
8730 return;
8731 }
8732 case Intrinsic::vector_insert: {
8733 SDValue Vec = getValue(I.getOperand(0));
8734 SDValue SubVec = getValue(I.getOperand(1));
8735 SDValue Index = getValue(I.getOperand(2));
8736
8737 // The intrinsic's index type is i64, but the SDNode requires an index type
8738 // suitable for the target. Convert the index as required.
8739 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8740 if (Index.getValueType() != VectorIdxTy)
8741 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8742
8743 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8744 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
8745 Index));
8746 return;
8747 }
8748 case Intrinsic::vector_extract: {
8749 SDValue Vec = getValue(I.getOperand(0));
8750 SDValue Index = getValue(I.getOperand(1));
8751 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8752
8753 // The intrinsic's index type is i64, but the SDNode requires an index type
8754 // suitable for the target. Convert the index as required.
8755 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8756 if (Index.getValueType() != VectorIdxTy)
8757 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8758
8759 setValue(&I,
8760 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
8761 return;
8762 }
8763 case Intrinsic::experimental_vector_match: {
8764 SDValue Op1 = getValue(I.getOperand(0));
8765 SDValue Op2 = getValue(I.getOperand(1));
8766 SDValue Mask = getValue(I.getOperand(2));
8767 EVT ResVT = Mask.getValueType();
8768 setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
8769 return;
8770 }
8771 case Intrinsic::vector_reverse:
8772 visitVectorReverse(I);
8773 return;
8774 case Intrinsic::vector_splice_left:
8775 case Intrinsic::vector_splice_right:
8776 visitVectorSplice(I);
8777 return;
8778 case Intrinsic::callbr_landingpad:
8779 visitCallBrLandingPad(I);
8780 return;
8781 case Intrinsic::vector_interleave2:
8782 visitVectorInterleave(I, 2);
8783 return;
8784 case Intrinsic::vector_interleave3:
8785 visitVectorInterleave(I, 3);
8786 return;
8787 case Intrinsic::vector_interleave4:
8788 visitVectorInterleave(I, 4);
8789 return;
8790 case Intrinsic::vector_interleave5:
8791 visitVectorInterleave(I, 5);
8792 return;
8793 case Intrinsic::vector_interleave6:
8794 visitVectorInterleave(I, 6);
8795 return;
8796 case Intrinsic::vector_interleave7:
8797 visitVectorInterleave(I, 7);
8798 return;
8799 case Intrinsic::vector_interleave8:
8800 visitVectorInterleave(I, 8);
8801 return;
8802 case Intrinsic::vector_deinterleave2:
8803 visitVectorDeinterleave(I, 2);
8804 return;
8805 case Intrinsic::vector_deinterleave3:
8806 visitVectorDeinterleave(I, 3);
8807 return;
8808 case Intrinsic::vector_deinterleave4:
8809 visitVectorDeinterleave(I, 4);
8810 return;
8811 case Intrinsic::vector_deinterleave5:
8812 visitVectorDeinterleave(I, 5);
8813 return;
8814 case Intrinsic::vector_deinterleave6:
8815 visitVectorDeinterleave(I, 6);
8816 return;
8817 case Intrinsic::vector_deinterleave7:
8818 visitVectorDeinterleave(I, 7);
8819 return;
8820 case Intrinsic::vector_deinterleave8:
8821 visitVectorDeinterleave(I, 8);
8822 return;
8823 case Intrinsic::vector_repeat: {
8824 SDValue Vec = getValue(I.getOperand(0));
8825 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8826 setValue(&I, DAG.getNode(ISD::VECTOR_REPEAT, sdl, ResultVT, Vec));
8827 return;
8828 }
8829 case Intrinsic::experimental_vector_compress:
8830 setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,
8831 getValue(I.getArgOperand(0)).getValueType(),
8832 getValue(I.getArgOperand(0)),
8833 getValue(I.getArgOperand(1)),
8834 getValue(I.getArgOperand(2)), Flags));
8835 return;
8836 case Intrinsic::vector_shuffle:
8837 setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, sdl,
8838 TLI.getValueType(DAG.getDataLayout(), I.getType()),
8839 getValue(I.getArgOperand(0)),
8840 getValue(I.getArgOperand(1)), Flags));
8841 return;
8842 case Intrinsic::experimental_convergence_anchor:
8843 case Intrinsic::experimental_convergence_entry:
8844 case Intrinsic::experimental_convergence_loop:
8845 visitConvergenceControl(I, Intrinsic);
8846 return;
8847 case Intrinsic::experimental_vector_histogram_add: {
8848 visitVectorHistogram(I, Intrinsic);
8849 return;
8850 }
8851 case Intrinsic::experimental_vector_extract_last_active: {
8852 visitVectorExtractLastActive(I, Intrinsic);
8853 return;
8854 }
8855 case Intrinsic::loop_dependence_war_mask:
8856 setValue(&I,
8858 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),
8859 getValue(I.getOperand(1)), getValue(I.getOperand(2)),
8860 DAG.getConstant(0, sdl, MVT::i64)));
8861 return;
8862 case Intrinsic::loop_dependence_raw_mask:
8863 setValue(&I,
8865 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),
8866 getValue(I.getOperand(1)), getValue(I.getOperand(2)),
8867 DAG.getConstant(0, sdl, MVT::i64)));
8868 return;
8869 case Intrinsic::masked_udiv:
8870 setValue(&I,
8871 DAG.getNode(ISD::MASKED_UDIV, sdl, EVT::getEVT(I.getType()),
8872 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8873 getValue(I.getOperand(2))));
8874 return;
8875 case Intrinsic::masked_sdiv:
8876 setValue(&I,
8877 DAG.getNode(ISD::MASKED_SDIV, sdl, EVT::getEVT(I.getType()),
8878 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8879 getValue(I.getOperand(2))));
8880 return;
8881 case Intrinsic::masked_urem:
8882 setValue(&I,
8883 DAG.getNode(ISD::MASKED_UREM, sdl, EVT::getEVT(I.getType()),
8884 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8885 getValue(I.getOperand(2))));
8886 return;
8887 case Intrinsic::masked_srem:
8888 setValue(&I,
8889 DAG.getNode(ISD::MASKED_SREM, sdl, EVT::getEVT(I.getType()),
8890 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8891 getValue(I.getOperand(2))));
8892 return;
8893 }
8894}
8895
8896void SelectionDAGBuilder::pushFPOpOutChain(SDValue Result,
8898 assert(Result.getNode()->getNumValues() == 2);
8899 SDValue OutChain = Result.getValue(1);
8900 assert(OutChain.getValueType() == MVT::Other);
8901
8902 // Instead of updating the root immediately, push the produced chain to the
8903 // appropriate list, deferring the update until the root is requested. In this
8904 // case, the nodes from the lists are chained using TokenFactor, indicating
8905 // that the operations are independent.
8906 //
8907 // In particular, the root is updated before any call that might access the
8908 // floating-point environment, except for constrained intrinsics.
8909 switch (EB) {
8912 PendingConstrainedFP.push_back(OutChain);
8913 break;
8915 PendingConstrainedFPStrict.push_back(OutChain);
8916 break;
8917 }
8918}
8919
8920void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8921 const ConstrainedFPIntrinsic &FPI) {
8922 SDLoc sdl = getCurSDLoc();
8923
8924 // We do not need to serialize constrained FP intrinsics against
8925 // each other or against (nonvolatile) loads, so they can be
8926 // chained like loads.
8928 SDValue Chain = getFPOperationRoot(EB);
8930 Opers.push_back(Chain);
8931 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8932 Opers.push_back(getValue(FPI.getArgOperand(I)));
8933
8934 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8935 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
8936 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
8937
8938 SDNodeFlags Flags;
8940 Flags.setNoFPExcept(true);
8941
8942 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
8943 Flags.copyFMF(*FPOp);
8944
8945 unsigned Opcode;
8946 switch (FPI.getIntrinsicID()) {
8947 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8948#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8949 case Intrinsic::INTRINSIC: \
8950 Opcode = ISD::STRICT_##DAGN; \
8951 break;
8952#include "llvm/IR/ConstrainedOps.def"
8953 case Intrinsic::experimental_constrained_fmuladd: {
8954 Opcode = ISD::STRICT_FMA;
8955 // Break fmuladd into fmul and fadd.
8956 if (!TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
8957 Opers.pop_back();
8958 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8959 pushFPOpOutChain(Mul, EB);
8960 Opcode = ISD::STRICT_FADD;
8961 Opers.clear();
8962 Opers.push_back(Mul.getValue(1));
8963 Opers.push_back(Mul.getValue(0));
8964 Opers.push_back(getValue(FPI.getArgOperand(2)));
8965 }
8966 break;
8967 }
8968 }
8969
8970 // A few strict DAG nodes carry additional operands that are not
8971 // set up by the default code above.
8972 switch (Opcode) {
8973 default: break;
8975 Opers.push_back(
8976 DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
8977 break;
8978 case ISD::STRICT_FSETCC:
8979 case ISD::STRICT_FSETCCS: {
8980 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8981 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8982 if (DAG.isKnownNeverNaN(Opers[1]) && DAG.isKnownNeverNaN(Opers[2]))
8983 Condition = getFCmpCodeWithoutNaN(Condition);
8984 Opers.push_back(DAG.getCondCode(Condition));
8985 break;
8986 }
8987 }
8988
8989 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8990 pushFPOpOutChain(Result, EB);
8991
8992 SDValue FPResult = Result.getValue(0);
8993 setValue(&FPI, FPResult);
8994}
8995
8996static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8997 std::optional<unsigned> ResOPC;
8998 switch (VPIntrin.getIntrinsicID()) {
8999 case Intrinsic::vp_cttz_elts: {
9000 bool IsZeroPoison = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
9001 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_POISON : ISD::VP_CTTZ_ELTS;
9002 break;
9003 }
9004#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
9005 case Intrinsic::VPID: \
9006 ResOPC = ISD::VPSD; \
9007 break;
9008#include "llvm/IR/VPIntrinsics.def"
9009 }
9010
9011 if (!ResOPC)
9013 "Inconsistency: no SDNode available for this VPIntrinsic!");
9014
9015 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
9016 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
9017 if (VPIntrin.getFastMathFlags().allowReassoc())
9018 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
9019 : ISD::VP_REDUCE_FMUL;
9020 }
9021
9022 return *ResOPC;
9023}
9024
9025void SelectionDAGBuilder::visitVPLoad(
9026 const VPIntrinsic &VPIntrin, EVT VT,
9027 const SmallVectorImpl<SDValue> &OpValues) {
9028 SDLoc DL = getCurSDLoc();
9029 Value *PtrOperand = VPIntrin.getArgOperand(0);
9030 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9031 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9032 const MDNode *Ranges = getRangeMetadata(VPIntrin);
9033 SDValue LD;
9034 // Do not serialize variable-length loads of constant memory with
9035 // anything.
9036 if (!Alignment)
9037 Alignment = DAG.getEVTAlign(VT);
9038 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
9039 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
9040 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9041 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9042 MachineMemOperand::Flags MMOFlags =
9044 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9045 MachinePointerInfo(PtrOperand), MMOFlags,
9047 MMOMetadata(AAInfo, Ranges));
9048 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9049 MMO, false /*IsExpanding */);
9050 if (AddToChain)
9051 PendingLoads.push_back(LD.getValue(1));
9052 setValue(&VPIntrin, LD);
9053}
9054
9055void SelectionDAGBuilder::visitVPLoadFF(
9056 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,
9057 const SmallVectorImpl<SDValue> &OpValues) {
9058 assert(OpValues.size() == 3 && "Unexpected number of operands");
9059 SDLoc DL = getCurSDLoc();
9060 Value *PtrOperand = VPIntrin.getArgOperand(0);
9061 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9062 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9063 const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
9064 SDValue LD;
9065 // Do not serialize variable-length loads of constant memory with
9066 // anything.
9067 if (!Alignment)
9068 Alignment = DAG.getEVTAlign(VT);
9069 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
9070 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
9071 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9072 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9073 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
9075 MMOMetadata(AAInfo, Ranges));
9076 LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9077 MMO);
9078 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));
9079 if (AddToChain)
9080 PendingLoads.push_back(LD.getValue(2));
9081 setValue(&VPIntrin, DAG.getMergeValues({LD.getValue(0), Trunc}, DL));
9082}
9083
9084void SelectionDAGBuilder::visitVPGather(
9085 const VPIntrinsic &VPIntrin, EVT VT,
9086 const SmallVectorImpl<SDValue> &OpValues) {
9087 SDLoc DL = getCurSDLoc();
9088 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9089 Value *PtrOperand = VPIntrin.getArgOperand(0);
9090 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9091 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9092 const MDNode *Ranges = getRangeMetadata(VPIntrin);
9093 SDValue LD;
9094 if (!Alignment)
9095 Alignment = DAG.getEVTAlign(VT.getScalarType());
9096 unsigned AS =
9097 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9098 MachineMemOperand::Flags MMOFlags =
9100 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9101 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9102 *Alignment, MMOMetadata(AAInfo, Ranges));
9103 SDValue Base, Index, Scale;
9104 bool UniformBase =
9105 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
9106 VT.getScalarStoreSize());
9107 if (!UniformBase) {
9108 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
9109 Index = getValue(PtrOperand);
9110 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
9111 }
9112 EVT IdxVT = Index.getValueType();
9113 EVT EltTy = IdxVT.getVectorElementType();
9114 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
9115 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
9116 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
9117 }
9118 LD = DAG.getGatherVP(
9119 DAG.getVTList(VT, MVT::Other), VT, DL,
9120 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
9122 PendingLoads.push_back(LD.getValue(1));
9123 setValue(&VPIntrin, LD);
9124}
9125
9126void SelectionDAGBuilder::visitVPStore(
9127 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9128 SDLoc DL = getCurSDLoc();
9129 Value *PtrOperand = VPIntrin.getArgOperand(1);
9130 EVT VT = OpValues[0].getValueType();
9131 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9132 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9133 SDValue ST;
9134 if (!Alignment)
9135 Alignment = DAG.getEVTAlign(VT);
9136 SDValue Ptr = OpValues[1];
9137 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
9138 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9139 MachineMemOperand::Flags MMOFlags =
9141 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9142 MachinePointerInfo(PtrOperand), MMOFlags,
9143 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
9144 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
9145 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
9146 /* IsTruncating */ false, /*IsCompressing*/ false);
9147 DAG.setRoot(ST);
9148 setValue(&VPIntrin, ST);
9149}
9150
9151void SelectionDAGBuilder::visitVPScatter(
9152 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9153 SDLoc DL = getCurSDLoc();
9154 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9155 Value *PtrOperand = VPIntrin.getArgOperand(1);
9156 EVT VT = OpValues[0].getValueType();
9157 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9158 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9159 SDValue ST;
9160 if (!Alignment)
9161 Alignment = DAG.getEVTAlign(VT.getScalarType());
9162 unsigned AS =
9163 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9164 MachineMemOperand::Flags MMOFlags =
9166 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9167 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9168 *Alignment, AAInfo);
9169 SDValue Base, Index, Scale;
9170 bool UniformBase =
9171 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
9172 VT.getScalarStoreSize());
9173 if (!UniformBase) {
9174 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
9175 Index = getValue(PtrOperand);
9176 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
9177 }
9178 EVT IdxVT = Index.getValueType();
9179 EVT EltTy = IdxVT.getVectorElementType();
9180 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
9181 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
9182 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
9183 }
9184 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
9185 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
9186 OpValues[2], OpValues[3]},
9187 MMO, ISD::SIGNED_SCALED);
9188 DAG.setRoot(ST);
9189 setValue(&VPIntrin, ST);
9190}
9191
9192void SelectionDAGBuilder::visitVPStridedLoad(
9193 const VPIntrinsic &VPIntrin, EVT VT,
9194 const SmallVectorImpl<SDValue> &OpValues) {
9195 SDLoc DL = getCurSDLoc();
9196 Value *PtrOperand = VPIntrin.getArgOperand(0);
9197 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9198 if (!Alignment)
9199 Alignment = DAG.getEVTAlign(VT.getScalarType());
9200 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9201 const MDNode *Ranges = getRangeMetadata(VPIntrin);
9202 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
9203 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
9204 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9205 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9206 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9207 MachineMemOperand::Flags MMOFlags =
9209 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9210 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9211 *Alignment, MMOMetadata(AAInfo, Ranges));
9212
9213 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
9214 OpValues[2], OpValues[3], MMO,
9215 false /*IsExpanding*/);
9216
9217 if (AddToChain)
9218 PendingLoads.push_back(LD.getValue(1));
9219 setValue(&VPIntrin, LD);
9220}
9221
9222void SelectionDAGBuilder::visitVPStridedStore(
9223 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9224 SDLoc DL = getCurSDLoc();
9225 Value *PtrOperand = VPIntrin.getArgOperand(1);
9226 EVT VT = OpValues[0].getValueType();
9227 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9228 if (!Alignment)
9229 Alignment = DAG.getEVTAlign(VT.getScalarType());
9230 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9231 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9232 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9233 MachineMemOperand::Flags MMOFlags =
9235 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9236 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9237 *Alignment, AAInfo);
9238
9239 SDValue ST = DAG.getStridedStoreVP(
9240 getMemoryRoot(), DL, OpValues[0], OpValues[1],
9241 DAG.getPOISON(OpValues[1].getValueType()), OpValues[2], OpValues[3],
9242 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
9243 /*IsCompressing*/ false);
9244
9245 DAG.setRoot(ST);
9246 setValue(&VPIntrin, ST);
9247}
9248
9249void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
9250 const VPIntrinsic &VPIntrin) {
9251 SDLoc DL = getCurSDLoc();
9252 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
9253
9254 auto IID = VPIntrin.getIntrinsicID();
9255
9256 SmallVector<EVT, 4> ValueVTs;
9257 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9258 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
9259 SDVTList VTs = DAG.getVTList(ValueVTs);
9260
9261 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
9262
9263 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
9264 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
9265 "Unexpected target EVL type");
9266
9267 // Request operands.
9268 SmallVector<SDValue, 7> OpValues;
9269 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
9270 auto Op = getValue(VPIntrin.getArgOperand(I));
9271 if (I == EVLParamPos)
9272 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
9273 OpValues.push_back(Op);
9274 }
9275
9276 switch (Opcode) {
9277 default: {
9278 SDNodeFlags SDFlags;
9279 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
9280 SDFlags.copyFMF(*FPMO);
9281 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
9282 setValue(&VPIntrin, Result);
9283 break;
9284 }
9285 case ISD::VP_LOAD:
9286 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
9287 break;
9288 case ISD::VP_LOAD_FF:
9289 visitVPLoadFF(VPIntrin, ValueVTs[0], ValueVTs[1], OpValues);
9290 break;
9291 case ISD::VP_GATHER:
9292 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
9293 break;
9294 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9295 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
9296 break;
9297 case ISD::VP_STORE:
9298 visitVPStore(VPIntrin, OpValues);
9299 break;
9300 case ISD::VP_SCATTER:
9301 visitVPScatter(VPIntrin, OpValues);
9302 break;
9303 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9304 visitVPStridedStore(VPIntrin, OpValues);
9305 break;
9306 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
9307 case ISD::VP_CTTZ_ELTS: {
9308 SDValue Result =
9309 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
9310 setValue(&VPIntrin, Result);
9311 break;
9312 }
9313 }
9314}
9315
9317 const BasicBlock *EHPadBB,
9318 MCSymbol *&BeginLabel) {
9319 MachineFunction &MF = DAG.getMachineFunction();
9320
9321 // Skip emitting EH_LABEL on targets whose exception tables don't reference
9322 // them (32-bit x86 SEH, Wasm).
9324 FuncInfo.ExceptionModel)) {
9325 BeginLabel = nullptr;
9326 return Chain;
9327 }
9328
9329 // Insert a label before the invoke call to mark the try range. This can be
9330 // used to detect deletion of the invoke via the MachineModuleInfo.
9331 BeginLabel = MF.getContext().createTempSymbol();
9332
9333 // For SjLj, keep track of which landing pads go with which invokes
9334 // so as to maintain the ordering of pads in the LSDA.
9335 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
9336 if (CallSiteIndex) {
9337 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
9338 LPadToCallSiteMap[FuncInfo.getMBB(EHPadBB)].push_back(CallSiteIndex);
9339
9340 // Now that the call site is handled, stop tracking it.
9341 FuncInfo.setCurrentCallSite(0);
9342 }
9343
9344 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
9345}
9346
9347SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
9348 const BasicBlock *EHPadBB,
9349 MCSymbol *BeginLabel) {
9350 // No labels were emitted.
9351 if (!BeginLabel)
9352 return Chain;
9353
9355
9356 // Insert a label at the end of the invoke call to mark the try range. This
9357 // can be used to detect deletion of the invoke via the MachineModuleInfo.
9358 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
9359 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
9360
9361 // Inform MachineModuleInfo of range.
9363 // There is a platform (e.g. wasm) that uses funclet style IR but does not
9364 // actually use outlined funclets and their LSDA info style.
9365 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
9366 assert(II && "II should've been set");
9367 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
9368 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
9369 } else if (!isScopedEHPersonality(Pers)) {
9370 assert(EHPadBB);
9371 MF.addInvoke(FuncInfo.getMBB(EHPadBB), BeginLabel, EndLabel);
9372 }
9373
9374 return Chain;
9375}
9376
9377std::pair<SDValue, SDValue>
9379 const BasicBlock *EHPadBB) {
9380 MCSymbol *BeginLabel = nullptr;
9381
9382 if (EHPadBB) {
9383 // Both PendingLoads and PendingExports must be flushed here;
9384 // this call might not return.
9385 (void)getRoot();
9386 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
9387 CLI.setChain(getRoot());
9388 }
9389
9390 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9391 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
9392
9393 assert((CLI.IsTailCall || Result.second.getNode()) &&
9394 "Non-null chain expected with non-tail call!");
9395 assert((Result.second.getNode() || !Result.first.getNode()) &&
9396 "Null value expected with tail call!");
9397
9398 if (!Result.second.getNode()) {
9399 // As a special case, a null chain means that a tail call has been emitted
9400 // and the DAG root is already updated.
9401 HasTailCall = true;
9402
9403 // Since there's no actual continuation from this block, nothing can be
9404 // relying on us setting vregs for them.
9405 PendingExports.clear();
9406 } else {
9407 DAG.setRoot(Result.second);
9408 }
9409
9410 if (EHPadBB) {
9411 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
9412 BeginLabel));
9413 Result.second = getRoot();
9414 }
9415
9416 return Result;
9417}
9418
9420 bool isMustTailCall = CB.isMustTailCall();
9421
9422 // Avoid emitting tail calls in functions with the disable-tail-calls
9423 // attribute.
9424 const Function *Caller = CB.getParent()->getParent();
9425 if (!isMustTailCall &&
9426 Caller->getFnAttribute("disable-tail-calls").getValueAsBool())
9427 return false;
9428
9429 // We can't tail call inside a function with a swifterror argument. Lowering
9430 // does not support this yet. It would have to move into the swifterror
9431 // register before the call.
9432 if (DAG.hasSwiftErrorArg())
9433 return false;
9434
9435 // Check if target-independent constraints permit a tail call here.
9436 // Target-dependent constraints are checked within TLI->LowerCallTo.
9437 return isInTailCallPosition(CB, DAG.getTarget());
9438}
9439
9441 bool isTailCall, bool isMustTailCall,
9442 const BasicBlock *EHPadBB,
9443 const TargetLowering::PtrAuthInfo *PAI) {
9444 auto &DL = DAG.getDataLayout();
9445 FunctionType *FTy = CB.getFunctionType();
9446 Type *RetTy = CB.getType();
9447
9449 Args.reserve(CB.arg_size());
9450
9451 const Value *SwiftErrorVal = nullptr;
9452 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9453
9454 if (isTailCall)
9455 isTailCall = canTailCall(CB);
9456
9457 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
9458 const Value *V = *I;
9459
9460 // Skip empty types
9461 if (V->getType()->isEmptyTy())
9462 continue;
9463
9464 SDValue ArgNode = getValue(V);
9465 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
9466 Entry.setAttributes(&CB, I - CB.arg_begin());
9467
9468 // Use swifterror virtual register as input to the call.
9469 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
9470 SwiftErrorVal = V;
9471 // We find the virtual register for the actual swifterror argument.
9472 // Instead of using the Value, we use the virtual register instead.
9473 Entry.Node =
9474 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
9475 EVT(TLI.getPointerTy(DL)));
9476 }
9477
9478 Args.push_back(Entry);
9479
9480 // If we have an explicit sret argument that is an Instruction, (i.e., it
9481 // might point to function-local memory), we can't meaningfully tail-call.
9482 if (Entry.IsSRet && isa<Instruction>(V))
9483 isTailCall = false;
9484 }
9485
9486 // If call site has a cfguardtarget operand bundle, create and add an
9487 // additional ArgListEntry.
9488 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
9489 Value *V = Bundle->Inputs[0];
9491 Entry.IsCFGuardTarget = true;
9492 Args.push_back(Entry);
9493 }
9494
9495 // Disable tail calls if there is an swifterror argument. Targets have not
9496 // been updated to support tail calls.
9497 if (TLI.supportSwiftError() && SwiftErrorVal)
9498 isTailCall = false;
9499
9500 ConstantInt *CFIType = nullptr;
9501 if (CB.isIndirectCall()) {
9502 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
9503 if (!TLI.supportKCFIBundles())
9505 "Target doesn't support calls with kcfi operand bundles.");
9506 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
9507 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
9508 }
9509 }
9510
9511 SDValue ConvControlToken;
9512 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
9513 auto *Token = Bundle->Inputs[0].get();
9514 ConvControlToken = getValue(Token);
9515 }
9516
9517 GlobalValue *DeactivationSymbol = nullptr;
9519 DeactivationSymbol = cast<GlobalValue>(Bundle->Inputs[0].get());
9520 }
9521
9524 .setChain(getRoot())
9525 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
9526 .setTailCall(isTailCall)
9530 .setCFIType(CFIType)
9531 .setConvergenceControlToken(ConvControlToken)
9532 .setDeactivationSymbol(DeactivationSymbol);
9533
9534 // Set the pointer authentication info if we have it.
9535 if (PAI) {
9536 if (!TLI.supportPtrAuthBundles())
9538 "This target doesn't support calls with ptrauth operand bundles.");
9539 CLI.setPtrAuth(*PAI);
9540 }
9541
9542 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9543
9544 if (Result.first.getNode()) {
9545 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
9546 Result.first = lowerNoFPClassToAssertNoFPClass(DAG, CB, Result.first);
9547 setValue(&CB, Result.first);
9548 }
9549
9550 // The last element of CLI.InVals has the SDValue for swifterror return.
9551 // Here we copy it to a virtual register and update SwiftErrorMap for
9552 // book-keeping.
9553 if (SwiftErrorVal && TLI.supportSwiftError()) {
9554 // Get the last element of InVals.
9555 SDValue Src = CLI.InVals.back();
9556 Register VReg =
9557 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
9558 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
9559 DAG.setRoot(CopyNode);
9560 }
9561}
9562
9563static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
9564 SelectionDAGBuilder &Builder) {
9565 // Check to see if this load can be trivially constant folded, e.g. if the
9566 // input is from a string literal.
9567 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
9568 // Cast pointer to the type we really want to load.
9569 Type *LoadTy =
9570 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
9571 if (LoadVT.isVector())
9572 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
9573 if (const Constant *LoadCst =
9574 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
9575 LoadTy, Builder.DAG.getDataLayout()))
9576 return Builder.getValue(LoadCst);
9577 }
9578
9579 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9580 // still constant memory, the input chain can be the entry node.
9581 SDValue Root;
9582 bool ConstantMemory = false;
9583
9584 // Do not serialize (non-volatile) loads of constant memory with anything.
9585 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) {
9586 Root = Builder.DAG.getEntryNode();
9587 ConstantMemory = true;
9588 } else {
9589 // Do not serialize non-volatile loads against each other.
9590 Root = Builder.DAG.getRoot();
9591 }
9592
9593 SDValue Ptr = Builder.getValue(PtrVal);
9594 SDValue LoadVal =
9595 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
9596 MachinePointerInfo(PtrVal), Align(1));
9597
9598 if (!ConstantMemory)
9599 Builder.PendingLoads.push_back(LoadVal.getValue(1));
9600 return LoadVal;
9601}
9602
9603/// Record the value for an instruction that produces an integer result,
9604/// converting the type where necessary.
9605void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9606 SDValue Value,
9607 bool IsSigned) {
9608 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
9609 I.getType(), true);
9610 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
9611 setValue(&I, Value);
9612}
9613
9614/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9615/// true and lower it. Otherwise return false, and it will be lowered like a
9616/// normal call.
9617/// The caller already checked that \p I calls the appropriate LibFunc with a
9618/// correct prototype.
9619bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9620 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
9621 const Value *Size = I.getArgOperand(2);
9622 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
9623 if (CSize && CSize->getZExtValue() == 0) {
9624 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
9625 I.getType(), true);
9626 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
9627 return true;
9628 }
9629
9630 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9631 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9632 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
9633 getValue(Size), &I);
9634 if (Res.first.getNode()) {
9635 processIntegerCallValue(I, Res.first, true);
9636 PendingLoads.push_back(Res.second);
9637 return true;
9638 }
9639
9640 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9641 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9642 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
9643 return false;
9644
9645 // If the target has a fast compare for the given size, it will return a
9646 // preferred load type for that size. Require that the load VT is legal and
9647 // that the target supports unaligned loads of that type. Otherwise, return
9648 // INVALID.
9649 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9650 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9651 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9652 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9653 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9654 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9655 // TODO: Check alignment of src and dest ptrs.
9656 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9657 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9658 if (!TLI.isTypeLegal(LVT) ||
9659 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
9662 }
9663
9664 return LVT;
9665 };
9666
9667 // This turns into unaligned loads. We only do this if the target natively
9668 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9669 // we'll only produce a small number of byte loads.
9670 MVT LoadVT;
9671 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9672 switch (NumBitsToCompare) {
9673 default:
9674 return false;
9675 case 16:
9676 LoadVT = MVT::i16;
9677 break;
9678 case 32:
9679 LoadVT = MVT::i32;
9680 break;
9681 case 64:
9682 case 128:
9683 case 256:
9684 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9685 break;
9686 }
9687
9688 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9689 return false;
9690
9691 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
9692 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
9693
9694 // Bitcast to a wide integer type if the loads are vectors.
9695 if (LoadVT.isVector()) {
9696 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
9697 LoadL = DAG.getBitcast(CmpVT, LoadL);
9698 LoadR = DAG.getBitcast(CmpVT, LoadR);
9699 }
9700
9701 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
9702 processIntegerCallValue(I, Cmp, false);
9703 return true;
9704}
9705
9706/// See if we can lower a memchr call into an optimized form. If so, return
9707/// true and lower it. Otherwise return false, and it will be lowered like a
9708/// normal call.
9709/// The caller already checked that \p I calls the appropriate LibFunc with a
9710/// correct prototype.
9711bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9712 const Value *Src = I.getArgOperand(0);
9713 const Value *Char = I.getArgOperand(1);
9714 const Value *Length = I.getArgOperand(2);
9715
9716 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9717 std::pair<SDValue, SDValue> Res =
9718 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
9719 getValue(Src), getValue(Char), getValue(Length),
9720 MachinePointerInfo(Src));
9721 if (Res.first.getNode()) {
9722 setValue(&I, Res.first);
9723 PendingLoads.push_back(Res.second);
9724 return true;
9725 }
9726
9727 return false;
9728}
9729
9730/// See if we can lower a memccpy call into an optimized form. If so, return
9731/// true and lower it, otherwise return false and it will be lowered like a
9732/// normal call.
9733/// The caller already checked that \p I calls the appropriate LibFunc with a
9734/// correct prototype.
9735bool SelectionDAGBuilder::visitMemCCpyCall(const CallInst &I) {
9736 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9737 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemccpy(
9738 DAG, getCurSDLoc(), DAG.getRoot(), getValue(I.getArgOperand(0)),
9739 getValue(I.getArgOperand(1)), getValue(I.getArgOperand(2)),
9740 getValue(I.getArgOperand(3)), &I);
9741
9742 if (Res.first) {
9743 processIntegerCallValue(I, Res.first, true);
9744 PendingLoads.push_back(Res.second);
9745 return true;
9746 }
9747 return false;
9748}
9749
9750/// See if we can lower a mempcpy call into an optimized form. If so, return
9751/// true and lower it. Otherwise return false, and it will be lowered like a
9752/// normal call.
9753/// The caller already checked that \p I calls the appropriate LibFunc with a
9754/// correct prototype.
9755bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9756 SDValue Dst = getValue(I.getArgOperand(0));
9757 SDValue Src = getValue(I.getArgOperand(1));
9758 SDValue Size = getValue(I.getArgOperand(2));
9759
9760 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
9761 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
9762
9763 SDLoc sdl = getCurSDLoc();
9764
9765 // In the mempcpy context we need to pass in a false value for isTailCall
9766 // because the return pointer needs to be adjusted by the size of
9767 // the copied memory.
9768 SDValue Root = getMemoryRoot();
9769 SDValue MC = DAG.getMemcpy(
9770 Root, sdl, Dst, Src, Size, DstAlign, SrcAlign, false, false,
9771 /*CI=*/nullptr, std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
9772 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
9773 assert(MC.getNode() != nullptr &&
9774 "** memcpy should not be lowered as TailCall in mempcpy context **");
9775 DAG.setRoot(MC);
9776
9777 // Check if Size needs to be truncated or extended.
9778 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
9779
9780 // Adjust return pointer to point just past the last dst byte.
9781 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Dst, Size, sdl);
9782 setValue(&I, DstPlusSize);
9783 return true;
9784}
9785
9786/// See if we can lower a strcpy call into an optimized form. If so, return
9787/// true and lower it, otherwise return false and it will be lowered like a
9788/// normal call.
9789/// The caller already checked that \p I calls the appropriate LibFunc with a
9790/// correct prototype.
9791bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9792 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9793
9794 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9795 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcpy(
9796 DAG, getCurSDLoc(), getRoot(), getValue(Arg0), getValue(Arg1),
9797 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), isStpcpy, &I);
9798 if (Res.first.getNode()) {
9799 setValue(&I, Res.first);
9800 DAG.setRoot(Res.second);
9801 return true;
9802 }
9803
9804 return false;
9805}
9806
9807/// See if we can lower a strcmp call into an optimized form. If so, return
9808/// true and lower it, otherwise return false and it will be lowered like a
9809/// normal call.
9810/// The caller already checked that \p I calls the appropriate LibFunc with a
9811/// correct prototype.
9812bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9813 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9814
9815 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9816 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcmp(
9817 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), getValue(Arg1),
9818 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), &I);
9819 if (Res.first.getNode()) {
9820 processIntegerCallValue(I, Res.first, true);
9821 PendingLoads.push_back(Res.second);
9822 return true;
9823 }
9824
9825 return false;
9826}
9827
9828/// See if we can lower a strlen call into an optimized form. If so, return
9829/// true and lower it, otherwise return false and it will be lowered like a
9830/// normal call.
9831/// The caller already checked that \p I calls the appropriate LibFunc with a
9832/// correct prototype.
9833bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9834 const Value *Arg0 = I.getArgOperand(0);
9835
9836 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9837 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(
9838 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), &I);
9839 if (Res.first.getNode()) {
9840 processIntegerCallValue(I, Res.first, false);
9841 PendingLoads.push_back(Res.second);
9842 return true;
9843 }
9844
9845 return false;
9846}
9847
9848/// See if we can lower a strnlen call into an optimized form. If so, return
9849/// true and lower it, otherwise return false and it will be lowered like a
9850/// normal call.
9851/// The caller already checked that \p I calls the appropriate LibFunc with a
9852/// correct prototype.
9853bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9854 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9855
9856 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9857 std::pair<SDValue, SDValue> Res =
9858 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
9859 getValue(Arg0), getValue(Arg1),
9860 MachinePointerInfo(Arg0));
9861 if (Res.first.getNode()) {
9862 processIntegerCallValue(I, Res.first, false);
9863 PendingLoads.push_back(Res.second);
9864 return true;
9865 }
9866
9867 return false;
9868}
9869
9870/// See if we can lower a Strstr call into an optimized form. If so, return
9871/// true and lower it, otherwise return false and it will be lowered like a
9872/// normal call.
9873/// The caller already checked that \p I calls the appropriate LibFunc with a
9874/// correct prototype.
9875bool SelectionDAGBuilder::visitStrstrCall(const CallInst &I) {
9876 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9877 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9878 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrstr(
9879 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), getValue(Arg1), &I);
9880 if (Res.first) {
9881 processIntegerCallValue(I, Res.first, false);
9882 PendingLoads.push_back(Res.second);
9883 return true;
9884 }
9885 return false;
9886}
9887
9888/// See if we can lower a unary floating-point operation into an SDNode with
9889/// the specified Opcode. If so, return true and lower it, otherwise return
9890/// false and it will be lowered like a normal call.
9891/// The caller already checked that \p I calls the appropriate LibFunc with a
9892/// correct prototype.
9893bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9894 unsigned Opcode) {
9895 // We already checked this call's prototype; verify it doesn't modify errno.
9896 // Do not perform optimizations for call sites that require strict
9897 // floating-point semantics.
9898 if (!I.onlyReadsMemory() || I.isStrictFP())
9899 return false;
9900
9901 SDNodeFlags Flags;
9902 Flags.copyFMF(cast<FPMathOperator>(I));
9903
9904 SDValue Tmp = getValue(I.getArgOperand(0));
9905 setValue(&I,
9906 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
9907 return true;
9908}
9909
9910/// See if we can lower a binary floating-point operation into an SDNode with
9911/// the specified Opcode. If so, return true and lower it. Otherwise return
9912/// false, and it will be lowered like a normal call.
9913/// The caller already checked that \p I calls the appropriate LibFunc with a
9914/// correct prototype.
9915bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9916 unsigned Opcode) {
9917 // We already checked this call's prototype; verify it doesn't modify errno.
9918 // Do not perform optimizations for call sites that require strict
9919 // floating-point semantics.
9920 if (!I.onlyReadsMemory() || I.isStrictFP())
9921 return false;
9922
9923 SDNodeFlags Flags;
9924 Flags.copyFMF(cast<FPMathOperator>(I));
9925
9926 SDValue Tmp0 = getValue(I.getArgOperand(0));
9927 SDValue Tmp1 = getValue(I.getArgOperand(1));
9928 EVT VT = Tmp0.getValueType();
9929 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
9930 return true;
9931}
9932
9933void SelectionDAGBuilder::visitCall(const CallInst &I) {
9934 // Handle inline assembly differently.
9935 if (I.isInlineAsm()) {
9936 visitInlineAsm(I);
9937 return;
9938 }
9939
9941
9942 if (Function *F = I.getCalledFunction()) {
9943 if (F->isDeclaration()) {
9944 // Is this an LLVM intrinsic?
9945 if (unsigned IID = F->getIntrinsicID()) {
9946 visitIntrinsicCall(I, IID);
9947 return;
9948 }
9949 }
9950
9951 // Check for well-known libc/libm calls. If the function is internal, it
9952 // can't be a library call. Don't do the check if marked as nobuiltin for
9953 // some reason.
9954 // This code should not handle libcalls that are already canonicalized to
9955 // intrinsics by the middle-end.
9956 LibFunc Func = !I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName()
9957 ? LibInfo->getLibFunc(*F)
9958 : NotLibFunc;
9959 if (LibInfo->hasOptimizedCodeGen(Func)) {
9960 switch (Func) {
9961 default: break;
9962 case LibFunc_bcmp:
9963 if (visitMemCmpBCmpCall(I))
9964 return;
9965 break;
9966 case LibFunc_sin:
9967 case LibFunc_sinf:
9968 case LibFunc_sinl:
9969 if (visitUnaryFloatCall(I, ISD::FSIN))
9970 return;
9971 break;
9972 case LibFunc_cos:
9973 case LibFunc_cosf:
9974 case LibFunc_cosl:
9975 if (visitUnaryFloatCall(I, ISD::FCOS))
9976 return;
9977 break;
9978 case LibFunc_tan:
9979 case LibFunc_tanf:
9980 case LibFunc_tanl:
9981 if (visitUnaryFloatCall(I, ISD::FTAN))
9982 return;
9983 break;
9984 case LibFunc_asin:
9985 case LibFunc_asinf:
9986 case LibFunc_asinl:
9987 if (visitUnaryFloatCall(I, ISD::FASIN))
9988 return;
9989 break;
9990 case LibFunc_acos:
9991 case LibFunc_acosf:
9992 case LibFunc_acosl:
9993 if (visitUnaryFloatCall(I, ISD::FACOS))
9994 return;
9995 break;
9996 case LibFunc_atan:
9997 case LibFunc_atanf:
9998 case LibFunc_atanl:
9999 if (visitUnaryFloatCall(I, ISD::FATAN))
10000 return;
10001 break;
10002 case LibFunc_atan2:
10003 case LibFunc_atan2f:
10004 case LibFunc_atan2l:
10005 if (visitBinaryFloatCall(I, ISD::FATAN2))
10006 return;
10007 break;
10008 case LibFunc_sinh:
10009 case LibFunc_sinhf:
10010 case LibFunc_sinhl:
10011 if (visitUnaryFloatCall(I, ISD::FSINH))
10012 return;
10013 break;
10014 case LibFunc_cosh:
10015 case LibFunc_coshf:
10016 case LibFunc_coshl:
10017 if (visitUnaryFloatCall(I, ISD::FCOSH))
10018 return;
10019 break;
10020 case LibFunc_tanh:
10021 case LibFunc_tanhf:
10022 case LibFunc_tanhl:
10023 if (visitUnaryFloatCall(I, ISD::FTANH))
10024 return;
10025 break;
10026 case LibFunc_sqrt:
10027 case LibFunc_sqrtf:
10028 case LibFunc_sqrtl:
10029 case LibFunc_sqrt_finite:
10030 case LibFunc_sqrtf_finite:
10031 case LibFunc_sqrtl_finite:
10032 if (visitUnaryFloatCall(I, ISD::FSQRT))
10033 return;
10034 break;
10035 case LibFunc_log2:
10036 case LibFunc_log2f:
10037 case LibFunc_log2l:
10038 if (visitUnaryFloatCall(I, ISD::FLOG2))
10039 return;
10040 break;
10041 case LibFunc_exp2:
10042 case LibFunc_exp2f:
10043 case LibFunc_exp2l:
10044 if (visitUnaryFloatCall(I, ISD::FEXP2))
10045 return;
10046 break;
10047 case LibFunc_exp10:
10048 case LibFunc_exp10f:
10049 case LibFunc_exp10l:
10050 if (visitUnaryFloatCall(I, ISD::FEXP10))
10051 return;
10052 break;
10053 case LibFunc_strstr:
10054 if (visitStrstrCall(I))
10055 return;
10056 break;
10057 case LibFunc_memcmp:
10058 if (visitMemCmpBCmpCall(I))
10059 return;
10060 break;
10061 case LibFunc_memccpy:
10062 if (visitMemCCpyCall(I))
10063 return;
10064 break;
10065 case LibFunc_mempcpy:
10066 if (visitMemPCpyCall(I))
10067 return;
10068 break;
10069 case LibFunc_memchr:
10070 if (visitMemChrCall(I))
10071 return;
10072 break;
10073 case LibFunc_strcpy:
10074 if (visitStrCpyCall(I, false))
10075 return;
10076 break;
10077 case LibFunc_stpcpy:
10078 if (visitStrCpyCall(I, true))
10079 return;
10080 break;
10081 case LibFunc_strcmp:
10082 if (visitStrCmpCall(I))
10083 return;
10084 break;
10085 case LibFunc_strlen:
10086 if (visitStrLenCall(I))
10087 return;
10088 break;
10089 case LibFunc_strnlen:
10090 if (visitStrNLenCall(I))
10091 return;
10092 break;
10093 }
10094 }
10095 }
10096
10097 if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
10098 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
10099 return;
10100 }
10101
10102 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
10103 // have to do anything here to lower funclet bundles.
10104 // CFGuardTarget bundles are lowered in LowerCallTo.
10106 I, "calls",
10111
10112 SDValue Callee = getValue(I.getCalledOperand());
10113
10114 if (I.hasDeoptState())
10115 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
10116 else
10117 // Check if we can potentially perform a tail call. More detailed checking
10118 // is be done within LowerCallTo, after more information about the call is
10119 // known.
10120 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
10121}
10122
10124 const CallBase &CB, const BasicBlock *EHPadBB) {
10125 auto PAB = CB.getOperandBundle("ptrauth");
10126 const Value *CalleeV = CB.getCalledOperand();
10127
10128 // Gather the call ptrauth data from the operand bundle:
10129 // [ i32 <key>, i64 <discriminator> ]
10130 const auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
10131 const Value *Discriminator = PAB->Inputs[1];
10132
10133 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
10134 assert(Discriminator->getType()->isIntegerTy(64) &&
10135 "Invalid ptrauth discriminator");
10136
10137 // Look through ptrauth constants to find the raw callee.
10138 // Do a direct unauthenticated call if we found it and everything matches.
10139 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CalleeV))
10140 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
10141 DAG.getDataLayout()))
10142 return LowerCallTo(CB, getValue(CalleeCPA->getPointer()), CB.isTailCall(),
10143 CB.isMustTailCall(), EHPadBB);
10144
10145 // Functions should never be ptrauth-called directly.
10146 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
10147
10148 // Otherwise, do an authenticated indirect call.
10149 TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
10150 getValue(Discriminator)};
10151
10152 LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
10153 EHPadBB, &PAI);
10154}
10155
10156namespace {
10157
10158/// AsmOperandInfo - This contains information for each constraint that we are
10159/// lowering.
10160class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
10161public:
10162 /// CallOperand - If this is the result output operand or a clobber
10163 /// this is null, otherwise it is the incoming operand to the CallInst.
10164 /// This gets modified as the asm is processed.
10165 SDValue CallOperand;
10166
10167 /// AssignedRegs - If this is a register or register class operand, this
10168 /// contains the set of register corresponding to the operand.
10169 RegsForValue AssignedRegs;
10170
10171 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
10172 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
10173 }
10174
10175 /// Whether or not this operand accesses memory
10176 bool hasMemory(const TargetLowering &TLI) const {
10177 // Indirect operand accesses access memory.
10178 if (isIndirect)
10179 return true;
10180
10181 for (const auto &Code : Codes)
10182 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
10183 return true;
10184
10185 return false;
10186 }
10187};
10188
10189
10190} // end anonymous namespace
10191
10192/// Make sure that the output operand \p OpInfo and its corresponding input
10193/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
10194/// out).
10195static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
10196 SDISelAsmOperandInfo &MatchingOpInfo,
10197 SelectionDAG &DAG) {
10198 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
10199 return;
10200
10202 const auto &TLI = DAG.getTargetLoweringInfo();
10203
10204 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
10205 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
10206 OpInfo.ConstraintVT);
10207 std::pair<unsigned, const TargetRegisterClass *> InputRC =
10208 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
10209 MatchingOpInfo.ConstraintVT);
10210 const bool OutOpIsIntOrFP =
10211 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
10212 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
10213 MatchingOpInfo.ConstraintVT.isFloatingPoint();
10214 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
10215 // FIXME: error out in a more elegant fashion
10216 report_fatal_error("Unsupported asm: input constraint"
10217 " with a matching output constraint of"
10218 " incompatible type!");
10219 }
10220 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
10221}
10222
10223/// Get a direct memory input to behave well as an indirect operand.
10224/// This may introduce stores, hence the need for a \p Chain.
10225/// \return The (possibly updated) chain.
10226static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
10227 SDISelAsmOperandInfo &OpInfo,
10228 SelectionDAG &DAG) {
10230
10231 // If we don't have an indirect input, put it in the constpool if we can,
10232 // otherwise spill it to a stack slot.
10233 // TODO: This isn't quite right. We need to handle these according to
10234 // the addressing mode that the constraint wants. Also, this may take
10235 // an additional register for the computation and we don't want that
10236 // either.
10237
10238 // If the operand is a float, integer, or vector constant, spill to a
10239 // constant pool entry to get its address.
10240 const Value *OpVal = OpInfo.CallOperandVal;
10241 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
10243 OpInfo.CallOperand = DAG.getConstantPool(
10244 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
10245 return Chain;
10246 }
10247
10248 // Otherwise, create a stack slot and emit a store to it before the asm.
10249 Type *Ty = OpVal->getType();
10250 auto &DL = DAG.getDataLayout();
10251 TypeSize TySize = DL.getTypeAllocSize(Ty);
10254 int StackID = 0;
10255 if (TySize.isScalable())
10256 StackID = TFI->getStackIDForScalableVectors();
10257 int SSFI = MF.getFrameInfo().CreateStackObject(TySize.getKnownMinValue(),
10258 DL.getPrefTypeAlign(Ty), false,
10259 nullptr, StackID);
10260 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
10261 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
10263 TLI.getMemValueType(DL, Ty));
10264 OpInfo.CallOperand = StackSlot;
10265
10266 return Chain;
10267}
10268
10269/// GetRegistersForValue - Assign registers (virtual or physical) for the
10270/// specified operand. We prefer to assign virtual registers, to allow the
10271/// register allocator to handle the assignment process. However, if the asm
10272/// uses features that we can't model on machineinstrs, we have SDISel do the
10273/// allocation. This produces generally horrible, but correct, code.
10274///
10275/// OpInfo describes the operand
10276/// RefOpInfo describes the matching operand if any, the operand otherwise
10277static std::optional<unsigned>
10279 SDISelAsmOperandInfo &OpInfo,
10280 SDISelAsmOperandInfo &RefOpInfo) {
10281 LLVMContext &Context = *DAG.getContext();
10283
10287
10288 // No work to do for memory/address operands.
10289 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10290 OpInfo.ConstraintType == TargetLowering::C_Address)
10291 return std::nullopt;
10292
10293 // If this is a constraint for a single physreg, or a constraint for a
10294 // register class, find it.
10295 unsigned AssignedReg;
10296 const TargetRegisterClass *RC;
10297 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
10298 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
10299 // RC is unset only on failure. Return immediately.
10300 if (!RC)
10301 return std::nullopt;
10302
10303 // Get the actual register value type. This is important, because the user
10304 // may have asked for (e.g.) the AX register in i32 type. We need to
10305 // remember that AX is actually i16 to get the right extension.
10306 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
10307
10308 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10309 // If this is an FP operand in an integer register (or visa versa), or more
10310 // generally if the operand value disagrees with the register class we plan
10311 // to stick it in, fix the operand type.
10312 //
10313 // If this is an input value, the bitcast to the new type is done now.
10314 // Bitcast for output value is done at the end of visitInlineAsm().
10315 if ((OpInfo.Type == InlineAsm::isOutput ||
10316 OpInfo.Type == InlineAsm::isInput) &&
10317 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
10318 // Try to convert to the first EVT that the reg class contains. If the
10319 // types are identical size, use a bitcast to convert (e.g. two differing
10320 // vector types). Note: output bitcast is done at the end of
10321 // visitInlineAsm().
10322 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10323 // Exclude indirect inputs while they are unsupported because the code
10324 // to perform the load is missing and thus OpInfo.CallOperand still
10325 // refers to the input address rather than the pointed-to value.
10326 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
10327 OpInfo.CallOperand =
10328 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
10329 OpInfo.ConstraintVT = RegVT;
10330 // If the operand is an FP value and we want it in integer registers,
10331 // use the corresponding integer type. This turns an f64 value into
10332 // i64, which can be passed with two i32 values on a 32-bit machine.
10333 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10334 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
10335 // No integer type of that size to bitcast to (e.g. f80).
10336 if (!VT.isValid())
10337 return std::nullopt;
10338 if (OpInfo.Type == InlineAsm::isInput)
10339 OpInfo.CallOperand =
10340 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
10341 OpInfo.ConstraintVT = VT;
10342 }
10343 }
10344 }
10345
10346 // No need to allocate a matching input constraint since the constraint it's
10347 // matching to has already been allocated.
10348 if (OpInfo.isMatchingInputConstraint())
10349 return std::nullopt;
10350
10351 EVT ValueVT = OpInfo.ConstraintVT;
10352 if (OpInfo.ConstraintVT == MVT::Other)
10353 ValueVT = RegVT;
10354
10355 // Initialize NumRegs.
10356 unsigned NumRegs = 1;
10357 if (OpInfo.ConstraintVT != MVT::Other)
10358 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
10359
10360 // If this is a constraint for a specific physical register, like {r17},
10361 // assign it now.
10362
10363 // If this associated to a specific register, initialize iterator to correct
10364 // place. If virtual, make sure we have enough registers
10365
10366 // Initialize iterator if necessary
10369
10370 // Do not check for single registers.
10371 if (AssignedReg) {
10372 I = std::find(I, RC->end(), AssignedReg);
10373 if (I == RC->end()) {
10374 // RC does not contain the selected register, which indicates a
10375 // mismatch between the register and the required type/bitwidth.
10376 return {AssignedReg};
10377 }
10378 }
10379
10380 for (; NumRegs; --NumRegs, ++I) {
10381 assert(I != RC->end() && "Ran out of registers to allocate!");
10382 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
10383 Regs.push_back(R);
10384 }
10385
10386 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
10387 return std::nullopt;
10388}
10389
10390static unsigned
10392 const std::vector<SDValue> &AsmNodeOperands) {
10393 // Scan until we find the definition we already emitted of this operand.
10394 unsigned CurOp = InlineAsm::Op_FirstOperand;
10395 for (; OperandNo; --OperandNo) {
10396 // Advance to the next operand.
10397 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10398 const InlineAsm::Flag F(OpFlag);
10399 assert(
10400 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
10401 "Skipped past definitions?");
10402 CurOp += F.getNumOperandRegisters() + 1;
10403 }
10404 return CurOp;
10405}
10406
10407namespace {
10408
10409class ExtraFlags {
10410 unsigned Flags = 0;
10411
10412public:
10413 explicit ExtraFlags(const CallBase &Call) {
10414 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
10415 if (IA->hasSideEffects())
10417 if (IA->isAlignStack())
10419 if (IA->canThrow())
10421 if (Call.isConvergent())
10423 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
10424 }
10425
10426 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
10427 // Ideally, we would only check against memory constraints. However, the
10428 // meaning of an Other constraint can be target-specific and we can't easily
10429 // reason about it. Therefore, be conservative and set MayLoad/MayStore
10430 // for Other constraints as well.
10433 if (OpInfo.Type == InlineAsm::isInput)
10435 else if (OpInfo.Type == InlineAsm::isOutput)
10437 else if (OpInfo.Type == InlineAsm::isClobber)
10439 }
10440 }
10441
10442 unsigned get() const { return Flags; }
10443};
10444
10445} // end anonymous namespace
10446
10447static bool isFunction(SDValue Op) {
10448 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
10449 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10450 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
10451
10452 // In normal "call dllimport func" instruction (non-inlineasm) it force
10453 // indirect access by specifing call opcode. And usually specially print
10454 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
10455 // not do in this way now. (In fact, this is similar with "Data Access"
10456 // action). So here we ignore dllimport function.
10457 if (Fn && !Fn->hasDLLImportStorageClass())
10458 return true;
10459 }
10460 }
10461 return false;
10462}
10463
10464namespace {
10465
10466struct ConstraintDecisionInfo {
10467 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
10468 std::vector<SDValue> AsmNodeOperands;
10469 SDValue Glue, Chain;
10470 bool HasSideEffect = false;
10471 MCSymbol *BeginLabel = nullptr;
10472
10473 SmallVector<char> Buffer;
10474 raw_svector_ostream ErrorMsg;
10475
10476 ConstraintDecisionInfo() : ErrorMsg(Buffer) {}
10477};
10478
10479} // end anonymous namespace
10480
10481/// Construct operand info objects.
10482static bool
10483constructOperandInfo(ConstraintDecisionInfo &Info,
10484 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10485 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10486 ExtraFlags &ExtraInfo) {
10487 for (auto &T : TargetConstraints) {
10488 Info.ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
10489 SDISelAsmOperandInfo &OpInfo = Info.ConstraintOperands.back();
10490
10491 if (OpInfo.CallOperandVal)
10492 OpInfo.CallOperand = Builder.getValue(OpInfo.CallOperandVal);
10493
10494 if (!Info.HasSideEffect)
10495 Info.HasSideEffect = OpInfo.hasMemory(TLI);
10496
10497 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
10498 // FIXME: Could we compute this on OpInfo rather than T?
10499
10500 // Compute the constraint code and ConstraintType to use.
10501 TLI.ComputeConstraintToUse(T, SDValue());
10502
10503 if (T.ConstraintType == TargetLowering::C_Immediate && OpInfo.CallOperand &&
10504 !isa<ConstantSDNode>(OpInfo.CallOperand)) {
10505 // We've delayed emitting a diagnostic like the "n" constraint because
10506 // inlining could cause an integer showing up.
10507 Info.ErrorMsg << "constraint '" << T.ConstraintCode
10508 << "' expects an integer constant expression";
10509 return true;
10510 }
10511
10512 ExtraInfo.update(T);
10513 }
10514
10515 return false;
10516}
10517
10518/// Compute which constraint option to use for each operand.
10519static void
10520computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
10521 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10522 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10523 const TargetMachine &TM, SelectionDAG &DAG) {
10524 const auto *IA = cast<InlineAsm>(Call.getCalledOperand());
10526 IA->collectAsmStrs(AsmStrs);
10527
10528 int OpNo = -1;
10529 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10530 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10531 OpNo++;
10532
10533 // If this is an output operand with a matching input operand, look up the
10534 // matching input. If their types mismatch, e.g. one is an integer, the
10535 // other is floating point, or their sizes are different, flag it as an
10536 // error.
10537 if (OpInfo.hasMatchingInput()) {
10538 SDISelAsmOperandInfo &Input =
10539 Info.ConstraintOperands[OpInfo.MatchingInput];
10540 patchMatchingInput(OpInfo, Input, DAG);
10541 }
10542
10543 // Compute the constraint code and ConstraintType to use.
10544 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
10545
10546 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10547 OpInfo.Type == InlineAsm::isClobber) ||
10548 OpInfo.ConstraintType == TargetLowering::C_Address)
10549 continue;
10550
10551 // In Linux PIC model, there are 4 cases about value/label addressing:
10552 //
10553 // 1: Function call or Label jmp inside the module.
10554 // 2: Data access (such as global variable, static variable) inside module.
10555 // 3: Function call or Label jmp outside the module.
10556 // 4: Data access (such as global variable) outside the module.
10557 //
10558 // Due to current llvm inline asm architecture designed to not "recognize"
10559 // the asm code, there are quite troubles for us to treat mem addressing
10560 // differently for same value/adress used in different instructions.
10561 // For example, in pic model, call a func may in plt way or direclty
10562 // pc-related, but lea/mov a function adress may use got.
10563 //
10564 // Here we try to "recognize" function call for the case 1 and case 3 in
10565 // inline asm. And try to adjust the constraint for them.
10566 //
10567 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10568 // label, so here we don't handle jmp function label now, but we need to
10569 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10570 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
10571 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10573 OpInfo.isIndirect = false;
10574 OpInfo.ConstraintType = TargetLowering::C_Address;
10575 }
10576
10577 // If this is a memory input, and if the operand is not indirect, do what we
10578 // need to provide an address for the memory input.
10579 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10580 !OpInfo.isIndirect) {
10581 assert((OpInfo.isMultipleAlternative ||
10582 (OpInfo.Type == InlineAsm::isInput)) &&
10583 "Can only indirectify direct input operands!");
10584
10585 // Memory operands really want the address of the value.
10586 Info.Chain = getAddressForMemoryInput(Info.Chain, Builder.getCurSDLoc(),
10587 OpInfo, DAG);
10588
10589 // There is no longer a Value* corresponding to this operand.
10590 OpInfo.CallOperandVal = nullptr;
10591
10592 // It is now an indirect operand.
10593 OpInfo.isIndirect = true;
10594 }
10595 }
10596}
10597
10598/// Prepare DAG-level operands. As part of this, assign virtual and physical
10599/// registers for inputs and output.
10600static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info,
10601 const CallBase &Call,
10602 SelectionDAGBuilder &Builder,
10603 const TargetLowering &TLI,
10604 SelectionDAG &DAG) {
10605 SDLoc DL = Builder.getCurSDLoc();
10606 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10607 // Assign Registers.
10608 SDISelAsmOperandInfo &RefOpInfo =
10609 OpInfo.isMatchingInputConstraint()
10610 ? Info.ConstraintOperands[OpInfo.getMatchedOperand()]
10611 : OpInfo;
10612 const auto RegError = getRegistersForValue(DAG, DL, OpInfo, RefOpInfo);
10613 if (RegError) {
10614 const MachineFunction &MF = DAG.getMachineFunction();
10616 const char *RegName = TRI.getName(*RegError);
10617 Info.ErrorMsg << "register '" << RegName << "' allocated for constraint '"
10618 << OpInfo.ConstraintCode
10619 << "' does not match required type";
10620 return true;
10621 }
10622
10623 auto DetectWriteToReservedRegister = [&]() {
10624 const MachineFunction &MF = DAG.getMachineFunction();
10626
10627 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10628 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
10629 Info.ErrorMsg << "write to reserved register '"
10630 << TRI.getRegAsmName(Reg) << "'";
10631 return true;
10632 }
10633 }
10634
10635 return false;
10636 };
10637 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10638 (OpInfo.Type == InlineAsm::isInput &&
10639 !OpInfo.isMatchingInputConstraint())) &&
10640 "Only address as input operand is allowed.");
10641
10642 switch (OpInfo.Type) {
10644 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10645 const InlineAsm::ConstraintCode ConstraintID =
10646 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10648 "Failed to convert memory constraint code to constraint id.");
10649
10650 // Add information to the INLINEASM node to know about this output.
10652 OpFlags.setMemConstraint(ConstraintID);
10653 Info.AsmNodeOperands.push_back(
10654 DAG.getTargetConstant(OpFlags, DL, MVT::i32));
10655 Info.AsmNodeOperands.push_back(OpInfo.CallOperand);
10656 } else {
10657 // Otherwise, this outputs to a register (directly for C_Register /
10658 // C_RegisterClass, and a target-defined fashion for
10659 // C_Immediate/C_Other). Find a register that we can use.
10660 if (OpInfo.AssignedRegs.Regs.empty()) {
10661 Info.ErrorMsg << "could not allocate output register for "
10662 << "constraint '" << OpInfo.ConstraintCode << "'";
10663 return true;
10664 }
10665
10666 if (DetectWriteToReservedRegister())
10667 return true;
10668
10669 // Add information to the INLINEASM node to know that this register is
10670 // set.
10671 OpInfo.AssignedRegs.AddInlineAsmOperands(
10672 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10674 false, 0, DL, DAG, Info.AsmNodeOperands);
10675 }
10676 break;
10677
10678 case InlineAsm::isInput:
10679 case InlineAsm::isLabel: {
10680 SDValue InOperandVal = OpInfo.CallOperand;
10681
10682 if (OpInfo.isMatchingInputConstraint()) {
10683 // If this is required to match an output register we have already set,
10684 // just use its register.
10685 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
10686 Info.AsmNodeOperands);
10687 InlineAsm::Flag Flag(Info.AsmNodeOperands[CurOp]->getAsZExtVal());
10688 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10689 if (OpInfo.isIndirect) {
10690 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10691 Info.ErrorMsg << "inline asm not supported yet: cannot handle "
10692 << "tied indirect register inputs";
10693 return true;
10694 }
10695
10700 auto *R = cast<RegisterSDNode>(Info.AsmNodeOperands[CurOp + 1]);
10701 Register TiedReg = R->getReg();
10702 MVT RegVT = R->getSimpleValueType(0);
10703 const TargetRegisterClass *RC;
10704 if (TiedReg.isVirtual()) {
10705 RC = MRI.getRegClass(TiedReg);
10706 } else if (RegVT == MVT::Untyped) {
10707 RC = TRI.getMinimalPhysRegClass(TiedReg);
10708 } else {
10709 RC = TLI.getRegClassFor(RegVT);
10710 if (!RC->contains(TiedReg)) {
10711 const TargetRegisterClass *PhysRC =
10712 TRI.getMinimalPhysRegClass(TiedReg);
10713 if (PhysRC->isAllocatable())
10714 RC = PhysRC;
10715 }
10716 }
10717
10718 for (unsigned I = 0, E = Flag.getNumOperandRegisters(); I != E; ++I)
10719 Regs.push_back(MRI.createVirtualRegister(RC));
10720
10721 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10722
10723 // Use the produced MatchedRegs object to
10724 MatchedRegs.getCopyToRegs(InOperandVal, DAG, DL, Info.Chain,
10725 &Info.Glue, &Call);
10727 OpInfo.getMatchedOperand(), DL, DAG,
10728 Info.AsmNodeOperands);
10729 break;
10730 }
10731
10732 assert(Flag.isMemKind() && "Unknown matching constraint!");
10733 assert(Flag.getNumOperandRegisters() == 1 &&
10734 "Unexpected number of operands");
10735
10736 // Add information to the INLINEASM node to know about this input.
10737 // See InlineAsm.h isUseOperandTiedToDef.
10738 Flag.clearMemConstraint();
10739 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10740 Info.AsmNodeOperands.push_back(DAG.getTargetConstant(
10741 Flag, DL, TLI.getPointerTy(DAG.getDataLayout())));
10742 Info.AsmNodeOperands.push_back(Info.AsmNodeOperands[CurOp + 1]);
10743 break;
10744 }
10745
10746 // Treat indirect 'X' constraint as memory.
10747 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10748 OpInfo.isIndirect)
10749 OpInfo.ConstraintType = TargetLowering::C_Memory;
10750
10751 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10752 OpInfo.ConstraintType == TargetLowering::C_Other) {
10753 std::vector<SDValue> Ops;
10754 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
10755 Ops, DAG);
10756 if (Ops.empty()) {
10757 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10758 if (isa<ConstantSDNode>(InOperandVal)) {
10759 Info.ErrorMsg << "value out of range for constraint '"
10760 << OpInfo.ConstraintCode << "'";
10761 return true;
10762 }
10763
10764 Info.ErrorMsg << "invalid operand for inline asm constraint '"
10765 << OpInfo.ConstraintCode << "'";
10766 return true;
10767 }
10768
10769 // Add information to the INLINEASM node to know about this input.
10770 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10771 Info.AsmNodeOperands.push_back(DAG.getTargetConstant(
10772 ResOpType, DL, TLI.getPointerTy(DAG.getDataLayout())));
10773 llvm::append_range(Info.AsmNodeOperands, Ops);
10774 break;
10775 }
10776
10777 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10778 assert((OpInfo.isIndirect ||
10779 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10780 "Operand must be indirect to be a mem!");
10781
10782 assert(InOperandVal.getValueType() ==
10783 TLI.getPointerTy(
10784 DAG.getDataLayout(),
10785 OpInfo.CallOperandVal
10786 ? cast<PointerType>(OpInfo.CallOperandVal->getType())
10787 ->getAddressSpace()
10788 : 0) &&
10789 "Memory operands expect pointer values");
10790
10791 const InlineAsm::ConstraintCode ConstraintID =
10792 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10794 "Failed to convert memory constraint code to constraint id.");
10795
10796 // Add information to the INLINEASM node to know about this input.
10798 ResOpType.setMemConstraint(ConstraintID);
10799 Info.AsmNodeOperands.push_back(
10800 DAG.getTargetConstant(ResOpType, DL, MVT::i32));
10801 Info.AsmNodeOperands.push_back(InOperandVal);
10802 break;
10803 }
10804
10805 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10806 const InlineAsm::ConstraintCode ConstraintID =
10807 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10809 "Failed to convert memory constraint code to constraint id.");
10810
10812
10813 SDValue AsmOp = InOperandVal;
10814 if (TLI.supportsInlineAsmFuncOperands() && isFunction(InOperandVal)) {
10815 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
10816 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10817 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
10818 InOperandVal.getValueType(),
10819 GA->getOffset());
10820 }
10821
10822 // Add information to the INLINEASM node to know about this input.
10823 ResOpType.setMemConstraint(ConstraintID);
10824
10825 Info.AsmNodeOperands.push_back(
10826 DAG.getTargetConstant(ResOpType, DL, MVT::i32));
10827 Info.AsmNodeOperands.push_back(AsmOp);
10828 break;
10829 }
10830
10831 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10832 OpInfo.ConstraintType != TargetLowering::C_Register) {
10833 Info.ErrorMsg << "unknown asm constraint '" << OpInfo.ConstraintCode
10834 << "'";
10835 return true;
10836 }
10837
10838 // TODO: Support this.
10839 if (OpInfo.isIndirect) {
10840 Info.ErrorMsg << "cannot handle indirect register inputs yet for "
10841 << "constraint '" << OpInfo.ConstraintCode << "'";
10842 return true;
10843 }
10844
10845 // Copy the input into the appropriate registers.
10846 if (OpInfo.AssignedRegs.Regs.empty()) {
10847 Info.ErrorMsg << "could not allocate input reg for constraint '"
10848 << OpInfo.ConstraintCode << "'";
10849 return true;
10850 }
10851
10852 if (DetectWriteToReservedRegister())
10853 return true;
10854
10855 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, DL, Info.Chain,
10856 &Info.Glue, &Call);
10857 OpInfo.AssignedRegs.AddInlineAsmOperands(
10858 InlineAsm::Kind::RegUse, false, 0, DL, DAG, Info.AsmNodeOperands);
10859 break;
10860 }
10861
10863 // Add the clobbered value to the operand list, so that the register
10864 // allocator is aware that the physreg got clobbered.
10865 if (!OpInfo.AssignedRegs.Regs.empty())
10866 OpInfo.AssignedRegs.AddInlineAsmOperands(
10867 InlineAsm::Kind::Clobber, false, 0, DL, DAG, Info.AsmNodeOperands);
10868 break;
10869 }
10870 }
10871
10872 return false;
10873}
10874
10875/// DetermineConstraints - Find the constraints to use for inline asm operands.
10876static bool
10877determineConstraints(ConstraintDecisionInfo &Info,
10878 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10879 const CallBase &Call, SelectionDAGBuilder &Builder,
10880 const TargetLowering &TLI, const TargetMachine &TM,
10881 SelectionDAG &DAG, const BasicBlock *EHPadBB) {
10882 const auto *IA = cast<InlineAsm>(Call.getCalledOperand());
10883 ExtraFlags ExtraInfo(Call);
10884
10885 // First pass: Construct operand info objects.
10886 Info.HasSideEffect = IA->hasSideEffects();
10887 if (constructOperandInfo(Info, TargetConstraints, Builder, TLI, ExtraInfo))
10888 return true;
10889
10890 // We won't need to flush pending loads if this asm doesn't touch
10891 // memory and is nonvolatile.
10892 Info.Chain = Info.HasSideEffect ? Builder.getRoot() : DAG.getRoot();
10893
10894 bool IsCallBr = isa<CallBrInst>(Call);
10895 bool EmitEHLabels = isa<InvokeInst>(Call);
10896 if (IsCallBr || EmitEHLabels)
10897 // If this is a callbr or invoke we need to flush pending exports since
10898 // inlineasm_br and invoke are terminators.
10899 // We need to do this before nodes are glued to the inlineasm_br node.
10900 Info.Chain = Builder.getControlRoot();
10901
10902 if (EmitEHLabels)
10903 Info.Chain = Builder.lowerStartEH(Info.Chain, EHPadBB, Info.BeginLabel);
10904
10905 // Second pass: Compute which constraint option to use.
10906 computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM, DAG);
10907
10908 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10909 Info.AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
10910 Info.AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
10911 IA->getAsmString().data(), TLI.getProgramPointerTy(DAG.getDataLayout())));
10912
10913 // If we have a !srcloc metadata node associated with it, we want to attach
10914 // this to the ultimately generated inline asm machineinstr. To do this, we
10915 // pass in the third operand as this (potentially null) inline asm MDNode.
10916 const MDNode *SrcLoc = Call.getMetadata("srcloc");
10917 Info.AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
10918
10919 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10920 // bits as operand 3.
10921 Info.AsmNodeOperands.push_back(
10922 DAG.getTargetConstant(ExtraInfo.get(), Builder.getCurSDLoc(),
10923 TLI.getPointerTy(DAG.getDataLayout())));
10924
10925 // Third pass: Prepare DAG-level operands
10926 return prepareDAGLevelOperands(Info, Call, Builder, TLI, DAG);
10927}
10928
10929/// visitInlineAsm - Handle a call to an InlineAsm object.
10930void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
10931 const BasicBlock *EHPadBB) {
10932 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10934 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);
10935
10936 assert((!isa<InvokeInst>(Call) || EHPadBB) &&
10937 "InvokeInst must have an EHPadBB");
10938
10939 ConstraintDecisionInfo Info;
10940 if (determineConstraints(Info, TargetConstraints, Call, *this, TLI, TM, DAG,
10941 EHPadBB))
10942 return emitInlineAsmError(Call, Info.ErrorMsg.str());
10943
10944 SDValue Glue = Info.Glue;
10945 SDValue Chain = Info.Chain;
10946
10947 // Finish up input operands. Set the input chain and add the flag last.
10948 Info.AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10949 if (Glue.getNode())
10950 Info.AsmNodeOperands.push_back(Glue);
10951
10952 bool IsCallBr = isa<CallBrInst>(Call);
10953 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10954 Chain =
10955 DAG.getNode(ISDOpc, getCurSDLoc(), DAG.getVTList(MVT::Other, MVT::Glue),
10956 Info.AsmNodeOperands);
10957 Glue = Chain.getValue(1);
10958
10959 // Do additional work to generate outputs.
10960
10961 SmallVector<EVT, 1> ResultVTs;
10962 SmallVector<SDValue, 1> ResultValues;
10963 SmallVector<SDValue, 8> OutChains;
10964
10965 llvm::Type *CallResultType = Call.getType();
10966 ArrayRef<Type *> ResultTypes;
10967 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
10968 ResultTypes = StructResult->elements();
10969 else if (!CallResultType->isVoidTy())
10970 ResultTypes = ArrayRef(CallResultType);
10971
10972 auto CurResultType = ResultTypes.begin();
10973 auto handleRegAssign = [&](SDValue V) {
10974 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10975 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10976 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
10977 ++CurResultType;
10978 // If the type of the inline asm call site return value is different but has
10979 // same size as the type of the asm output bitcast it. One example of this
10980 // is for vectors with different width / number of elements. This can
10981 // happen for register classes that can contain multiple different value
10982 // types. The preg or vreg allocated may not have the same VT as was
10983 // expected.
10984 //
10985 // This can also happen for a return value that disagrees with the register
10986 // class it is put in, eg. a double in a general-purpose register on a
10987 // 32-bit machine.
10988 if (ResultVT != V.getValueType() &&
10989 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10990 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
10991 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10992 V.getValueType().isInteger()) {
10993 // If a result value was tied to an input value, the computed result
10994 // may have a wider width than the expected result. Extract the
10995 // relevant portion.
10996 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
10997 }
10998 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10999 ResultVTs.push_back(ResultVT);
11000 ResultValues.push_back(V);
11001 };
11002
11003 // Deal with output operands.
11004 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
11005 if (OpInfo.Type == InlineAsm::isOutput) {
11006 SDValue Val;
11007 // Skip trivial output operands.
11008 if (OpInfo.AssignedRegs.Regs.empty())
11009 continue;
11010
11011 switch (OpInfo.ConstraintType) {
11014 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
11015 Chain, &Glue, &Call);
11016 break;
11019 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
11020 OpInfo, DAG);
11021 break;
11023 break; // Already handled.
11025 break; // Silence warning.
11027 assert(false && "Unexpected unknown constraint");
11028 }
11029
11030 // Indirect output manifest as stores. Record output chains.
11031 if (OpInfo.isIndirect) {
11032 const Value *Ptr = OpInfo.CallOperandVal;
11033 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
11034 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
11035 MachinePointerInfo(Ptr));
11036 OutChains.push_back(Store);
11037 } else {
11038 // generate CopyFromRegs to associated registers.
11039 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
11040 if (Val.getOpcode() == ISD::MERGE_VALUES) {
11041 for (const SDValue &V : Val->op_values())
11042 handleRegAssign(V);
11043 } else
11044 handleRegAssign(Val);
11045 }
11046 }
11047 }
11048
11049 // Set results.
11050 if (!ResultValues.empty()) {
11051 assert(CurResultType == ResultTypes.end() &&
11052 "Mismatch in number of ResultTypes");
11053 assert(ResultValues.size() == ResultTypes.size() &&
11054 "Mismatch in number of output operands in asm result");
11055
11057 DAG.getVTList(ResultVTs), ResultValues);
11058 setValue(&Call, V);
11059 }
11060
11061 // Collect store chains.
11062 if (!OutChains.empty())
11063 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
11064
11065 if (const auto *II = dyn_cast<InvokeInst>(&Call))
11066 Chain = lowerEndEH(Chain, II, EHPadBB, Info.BeginLabel);
11067
11068 // Only Update Root if inline assembly has a memory effect.
11069 if (ResultValues.empty() || Info.HasSideEffect || !OutChains.empty() ||
11070 IsCallBr || isa<InvokeInst>(Call))
11071 DAG.setRoot(Chain);
11072}
11073
11074void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
11075 const Twine &Message) {
11076 LLVMContext &Ctx = *DAG.getContext();
11077 Ctx.diagnose(DiagnosticInfoInlineAsm(Call, Message));
11078
11079 // Make sure we leave the DAG in a valid state
11080 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11081 SmallVector<EVT, 1> ValueVTs;
11082 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
11083
11084 if (ValueVTs.empty())
11085 return;
11086
11088 for (const EVT &VT : ValueVTs)
11089 Ops.push_back(DAG.getUNDEF(VT));
11090
11091 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
11092}
11093
11094void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
11095 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
11096 MVT::Other, getRoot(),
11097 getValue(I.getArgOperand(0)),
11098 DAG.getSrcValue(I.getArgOperand(0))));
11099}
11100
11101void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
11102 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11103 const DataLayout &DL = DAG.getDataLayout();
11104 SDValue V = DAG.getVAArg(
11105 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
11106 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
11107 DL.getABITypeAlign(I.getType()).value());
11108 DAG.setRoot(V.getValue(1));
11109
11110 if (I.getType()->isPointerTy())
11111 V = DAG.getPtrExtOrTrunc(
11112 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
11113 setValue(&I, V);
11114}
11115
11116void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
11117 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
11118 MVT::Other, getRoot(),
11119 getValue(I.getArgOperand(0)),
11120 DAG.getSrcValue(I.getArgOperand(0))));
11121}
11122
11123void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
11124 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
11125 MVT::Other, getRoot(),
11126 getValue(I.getArgOperand(0)),
11127 getValue(I.getArgOperand(1)),
11128 DAG.getSrcValue(I.getArgOperand(0)),
11129 DAG.getSrcValue(I.getArgOperand(1))));
11130}
11131
11133 const Instruction &I,
11134 SDValue Op) {
11135 std::optional<ConstantRange> CR = getRange(I);
11136
11137 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
11138 return Op;
11139
11140 APInt Hi = CR->getUnsignedMax();
11141 unsigned Bits = std::max(Hi.getActiveBits(),
11142 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
11143
11144 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
11145
11146 SDLoc SL = getCurSDLoc();
11147
11148 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
11149 DAG.getValueType(SmallVT));
11150 unsigned NumVals = Op.getNode()->getNumValues();
11151 if (NumVals == 1)
11152 return ZExt;
11153
11155
11156 Ops.push_back(ZExt);
11157 for (unsigned I = 1; I != NumVals; ++I)
11158 Ops.push_back(Op.getValue(I));
11159
11160 return DAG.getMergeValues(Ops, SL);
11161}
11162
11164 SelectionDAG &DAG, const Instruction &I, SDValue Op) {
11165 FPClassTest Classes = getNoFPClass(I);
11166 if (Classes == fcNone)
11167 return Op;
11168
11169 SDLoc SL = getCurSDLoc();
11170 SDValue TestConst = DAG.getTargetConstant(Classes, SDLoc(), MVT::i32);
11171
11172 if (Op.getOpcode() != ISD::MERGE_VALUES) {
11173 return DAG.getNode(ISD::AssertNoFPClass, SL, Op.getValueType(), Op,
11174 TestConst);
11175 }
11176
11177 SmallVector<SDValue, 8> Ops(Op.getNumOperands());
11178 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
11179 SDValue MergeOp = Op.getOperand(I);
11180 Ops[I] = DAG.getNode(ISD::AssertNoFPClass, SL, MergeOp.getValueType(),
11181 MergeOp, TestConst);
11182 }
11183
11184 return DAG.getMergeValues(Ops, SL);
11185}
11186
11187/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
11188/// the call being lowered.
11189///
11190/// This is a helper for lowering intrinsics that follow a target calling
11191/// convention or require stack pointer adjustment. Only a subset of the
11192/// intrinsic's operands need to participate in the calling convention.
11195 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
11196 AttributeSet RetAttrs, bool IsPatchPoint) {
11198 Args.reserve(NumArgs);
11199
11200 // Populate the argument list.
11201 // Attributes for args start at offset 1, after the return attribute.
11202 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
11203 ArgI != ArgE; ++ArgI) {
11204 const Value *V = Call->getOperand(ArgI);
11205
11206 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
11207
11208 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
11209 Entry.setAttributes(Call, ArgI);
11210 Args.push_back(Entry);
11211 }
11212
11214 .setChain(getRoot())
11215 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
11216 RetAttrs)
11217 .setDiscardResult(Call->use_empty())
11218 .setIsPatchPoint(IsPatchPoint)
11220 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
11221}
11222
11223/// Add a stack map intrinsic call's live variable operands to a stackmap
11224/// or patchpoint target node's operand list.
11225///
11226/// Constants are converted to TargetConstants purely as an optimization to
11227/// avoid constant materialization and register allocation.
11228///
11229/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
11230/// generate addess computation nodes, and so FinalizeISel can convert the
11231/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
11232/// address materialization and register allocation, but may also be required
11233/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
11234/// alloca in the entry block, then the runtime may assume that the alloca's
11235/// StackMap location can be read immediately after compilation and that the
11236/// location is valid at any point during execution (this is similar to the
11237/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
11238/// only available in a register, then the runtime would need to trap when
11239/// execution reaches the StackMap in order to read the alloca's location.
11240static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
11242 SelectionDAGBuilder &Builder) {
11243 SelectionDAG &DAG = Builder.DAG;
11244 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
11245 SDValue Op = Builder.getValue(Call.getArgOperand(I));
11246
11247 // Things on the stack are pointer-typed, meaning that they are already
11248 // legal and can be emitted directly to target nodes.
11250 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
11251 } else {
11252 // Otherwise emit a target independent node to be legalised.
11253 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
11254 }
11255 }
11256}
11257
11258/// Lower llvm.experimental.stackmap.
11259void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
11260 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
11261 // [live variables...])
11262
11263 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
11264
11265 SDValue Chain, InGlue, Callee;
11267
11268 SDLoc DL = getCurSDLoc();
11270
11271 // The stackmap intrinsic only records the live variables (the arguments
11272 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
11273 // intrinsic, this won't be lowered to a function call. This means we don't
11274 // have to worry about calling conventions and target specific lowering code.
11275 // Instead we perform the call lowering right here.
11276 //
11277 // chain, flag = CALLSEQ_START(chain, 0, 0)
11278 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
11279 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
11280 //
11281 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
11282 InGlue = Chain.getValue(1);
11283
11284 // Add the STACKMAP operands, starting with DAG house-keeping.
11285 Ops.push_back(Chain);
11286 Ops.push_back(InGlue);
11287
11288 // Add the <id>, <numShadowBytes> operands.
11289 //
11290 // These do not require legalisation, and can be emitted directly to target
11291 // constant nodes.
11293 assert(ID.getValueType() == MVT::i64);
11294 SDValue IDConst =
11295 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
11296 Ops.push_back(IDConst);
11297
11298 SDValue Shad = getValue(CI.getArgOperand(1));
11299 assert(Shad.getValueType() == MVT::i32);
11300 SDValue ShadConst =
11301 DAG.getTargetConstant(Shad->getAsZExtVal(), DL, Shad.getValueType());
11302 Ops.push_back(ShadConst);
11303
11304 // Add the live variables.
11305 addStackMapLiveVars(CI, 2, DL, Ops, *this);
11306
11307 // Create the STACKMAP node.
11308 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
11309 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
11310 InGlue = Chain.getValue(1);
11311
11312 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
11313
11314 // Stackmaps don't generate values, so nothing goes into the NodeMap.
11315
11316 // Set the root to the target-lowered call chain.
11317 DAG.setRoot(Chain);
11318
11319 // Inform the Frame Information that we have a stackmap in this function.
11320 FuncInfo.MF->getFrameInfo().setHasStackMap();
11321}
11322
11323/// Lower llvm.experimental.patchpoint directly to its target opcode.
11324void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
11325 const BasicBlock *EHPadBB) {
11326 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
11327 // i32 <numBytes>,
11328 // i8* <target>,
11329 // i32 <numArgs>,
11330 // [Args...],
11331 // [live variables...])
11332
11334 bool IsAnyRegCC = CC == CallingConv::AnyReg;
11335 bool HasDef = !CB.getType()->isVoidTy();
11336 SDLoc dl = getCurSDLoc();
11338
11339 // Handle immediate and symbolic callees.
11340 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
11341 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
11342 /*isTarget=*/true);
11343 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
11344 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
11345 SDLoc(SymbolicCallee),
11346 SymbolicCallee->getValueType(0));
11347
11348 // Get the real number of arguments participating in the call <numArgs>
11350 unsigned NumArgs = NArgVal->getAsZExtVal();
11351
11352 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
11353 // Intrinsics include all meta-operands up to but not including CC.
11354 unsigned NumMetaOpers = PatchPointOpers::CCPos;
11355 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
11356 "Not enough arguments provided to the patchpoint intrinsic");
11357
11358 // For AnyRegCC the arguments are lowered later on manually.
11359 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11360 Type *ReturnTy =
11361 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
11362
11363 TargetLowering::CallLoweringInfo CLI(DAG);
11364 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
11365 ReturnTy, CB.getAttributes().getRetAttrs(), true);
11366 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
11367
11368 SDNode *CallEnd = Result.second.getNode();
11369 if (CallEnd->getOpcode() == ISD::EH_LABEL)
11370 CallEnd = CallEnd->getOperand(0).getNode();
11371 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
11372 CallEnd = CallEnd->getOperand(0).getNode();
11373
11374 /// Get a call instruction from the call sequence chain.
11375 /// Tail calls are not allowed.
11376 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
11377 "Expected a callseq node.");
11378 SDNode *Call = CallEnd->getOperand(0).getNode();
11379 bool HasGlue = Call->getGluedNode();
11380
11381 // Replace the target specific call node with the patchable intrinsic.
11383
11384 // Push the chain.
11385 Ops.push_back(*(Call->op_begin()));
11386
11387 // Optionally, push the glue (if any).
11388 if (HasGlue)
11389 Ops.push_back(*(Call->op_end() - 1));
11390
11391 // Push the register mask info.
11392 if (HasGlue)
11393 Ops.push_back(*(Call->op_end() - 2));
11394 else
11395 Ops.push_back(*(Call->op_end() - 1));
11396
11397 // Add the <id> and <numBytes> constants.
11399 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
11401 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
11402
11403 // Add the callee.
11404 Ops.push_back(Callee);
11405
11406 // Adjust <numArgs> to account for any arguments that have been passed on the
11407 // stack instead.
11408 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
11409 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
11410 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11411 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
11412
11413 // Add the calling convention
11414 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
11415
11416 // Add the arguments we omitted previously. The register allocator should
11417 // place these in any free register.
11418 if (IsAnyRegCC)
11419 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
11420 Ops.push_back(getValue(CB.getArgOperand(i)));
11421
11422 // Push the arguments from the call instruction.
11423 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
11424 Ops.append(Call->op_begin() + 2, e);
11425
11426 // Push live variables for the stack map.
11427 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
11428
11429 SDVTList NodeTys;
11430 if (IsAnyRegCC && HasDef) {
11431 // Create the return types based on the intrinsic definition
11432 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11433 SmallVector<EVT, 3> ValueVTs;
11434 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
11435 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
11436
11437 // There is always a chain and a glue type at the end
11438 ValueVTs.push_back(MVT::Other);
11439 ValueVTs.push_back(MVT::Glue);
11440 NodeTys = DAG.getVTList(ValueVTs);
11441 } else
11442 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
11443
11444 // Replace the target specific call node with a PATCHPOINT node.
11445 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
11446
11447 // Update the NodeMap.
11448 if (HasDef) {
11449 if (IsAnyRegCC)
11450 setValue(&CB, SDValue(PPV.getNode(), 0));
11451 else
11452 setValue(&CB, Result.first);
11453 }
11454
11455 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
11456 // call sequence. Furthermore the location of the chain and glue can change
11457 // when the AnyReg calling convention is used and the intrinsic returns a
11458 // value.
11459 if (IsAnyRegCC && HasDef) {
11460 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
11461 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
11462 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
11463 } else
11464 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
11465 DAG.DeleteNode(Call);
11466
11467 // Inform the Frame Information that we have a patchpoint in this function.
11468 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11469}
11470
11471void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
11472 unsigned Intrinsic) {
11473 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11474 SDValue Op1 = getValue(I.getArgOperand(0));
11475 SDValue Op2;
11476 if (I.arg_size() > 1)
11477 Op2 = getValue(I.getArgOperand(1));
11478 SDLoc dl = getCurSDLoc();
11479 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11480 SDValue Res;
11481 SDNodeFlags SDFlags;
11482 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
11483 SDFlags.copyFMF(*FPMO);
11484
11485 switch (Intrinsic) {
11486 case Intrinsic::vector_reduce_fadd:
11487 if (SDFlags.hasAllowReassociation())
11488 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
11489 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
11490 SDFlags);
11491 else
11492 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
11493 break;
11494 case Intrinsic::vector_reduce_fmul:
11495 if (SDFlags.hasAllowReassociation())
11496 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
11497 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
11498 SDFlags);
11499 else
11500 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
11501 break;
11502 case Intrinsic::vector_reduce_add:
11503 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
11504 break;
11505 case Intrinsic::vector_reduce_mul:
11506 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
11507 break;
11508 case Intrinsic::vector_reduce_and:
11509 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
11510 break;
11511 case Intrinsic::vector_reduce_or:
11512 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
11513 break;
11514 case Intrinsic::vector_reduce_xor:
11515 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
11516 break;
11517 case Intrinsic::vector_reduce_smax:
11518 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
11519 break;
11520 case Intrinsic::vector_reduce_smin:
11521 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
11522 break;
11523 case Intrinsic::vector_reduce_umax:
11524 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
11525 break;
11526 case Intrinsic::vector_reduce_umin:
11527 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
11528 break;
11529 case Intrinsic::vector_reduce_fmax:
11530 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
11531 break;
11532 case Intrinsic::vector_reduce_fmin:
11533 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
11534 break;
11535 case Intrinsic::vector_reduce_fmaximum:
11536 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
11537 break;
11538 case Intrinsic::vector_reduce_fminimum:
11539 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
11540 break;
11541 case Intrinsic::vector_reduce_fmaximumnum:
11542 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUMNUM, dl, VT, Op1, SDFlags);
11543 break;
11544 case Intrinsic::vector_reduce_fminimumnum:
11545 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUMNUM, dl, VT, Op1, SDFlags);
11546 break;
11547 default:
11548 llvm_unreachable("Unhandled vector reduce intrinsic");
11549 }
11550 setValue(&I, Res);
11551}
11552
11553/// Returns an AttributeList representing the attributes applied to the return
11554/// value of the given call.
11557 if (CLI.RetSExt)
11558 Attrs.push_back(Attribute::SExt);
11559 if (CLI.RetZExt)
11560 Attrs.push_back(Attribute::ZExt);
11561 if (CLI.IsInReg)
11562 Attrs.push_back(Attribute::InReg);
11563
11564 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
11565 Attrs);
11566}
11567
11568/// TargetLowering::LowerCallTo - This is the default LowerCallTo
11569/// implementation, which just calls LowerCall.
11570/// FIXME: When all targets are
11571/// migrated to using LowerCall, this hook should be integrated into SDISel.
11572std::pair<SDValue, SDValue>
11574 LLVMContext &Context = CLI.RetTy->getContext();
11575
11576 // Handle the incoming return values from the call.
11577 CLI.Ins.clear();
11578 SmallVector<Type *, 4> RetOrigTys;
11580 auto &DL = CLI.DAG.getDataLayout();
11581 ComputeValueTypes(DL, CLI.OrigRetTy, RetOrigTys, &Offsets);
11582
11583 SmallVector<EVT, 4> RetVTs;
11584 if (CLI.RetTy != CLI.OrigRetTy) {
11585 assert(RetOrigTys.size() == 1 &&
11586 "Only supported for non-aggregate returns");
11587 RetVTs.push_back(getValueType(DL, CLI.RetTy));
11588 } else {
11589 for (Type *Ty : RetOrigTys)
11590 RetVTs.push_back(getValueType(DL, Ty));
11591 }
11592
11593 if (CLI.IsPostTypeLegalization) {
11594 // If we are lowering a libcall after legalization, split the return type.
11595 SmallVector<Type *, 4> OldRetOrigTys;
11596 SmallVector<EVT, 4> OldRetVTs;
11597 SmallVector<TypeSize, 4> OldOffsets;
11598 RetOrigTys.swap(OldRetOrigTys);
11599 RetVTs.swap(OldRetVTs);
11600 Offsets.swap(OldOffsets);
11601
11602 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
11603 EVT RetVT = OldRetVTs[i];
11604 uint64_t Offset = OldOffsets[i];
11605 MVT RegisterVT = getRegisterType(Context, RetVT);
11606 unsigned NumRegs = getNumRegisters(Context, RetVT);
11607 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
11608 RetOrigTys.append(NumRegs, OldRetOrigTys[i]);
11609 RetVTs.append(NumRegs, RegisterVT);
11610 for (unsigned j = 0; j != NumRegs; ++j)
11611 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
11612 }
11613 }
11614
11616 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
11617
11618 bool CanLowerReturn =
11620 CLI.IsVarArg, Outs, Context, CLI.RetTy);
11621
11622 SDValue DemoteStackSlot;
11623 int DemoteStackIdx = -100;
11624 if (!CanLowerReturn) {
11625 // FIXME: equivalent assert?
11626 // assert(!CS.hasInAllocaArgument() &&
11627 // "sret demotion is incompatible with inalloca");
11628 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
11629 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
11631 DemoteStackIdx =
11632 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
11633 Type *StackSlotPtrType = PointerType::get(Context, DL.getAllocaAddrSpace());
11634
11635 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
11636 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11637 Entry.IsSRet = true;
11638 Entry.Alignment = Alignment;
11639 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
11640 CLI.NumFixedArgs += 1;
11641 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11642 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(Context);
11643
11644 // sret demotion isn't compatible with tail-calls, since the sret argument
11645 // points into the callers stack frame.
11646 CLI.IsTailCall = false;
11647 } else {
11648 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11649 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
11650 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
11651 ISD::ArgFlagsTy Flags;
11652 if (NeedsRegBlock) {
11653 Flags.setInConsecutiveRegs();
11654 if (I == RetVTs.size() - 1)
11655 Flags.setInConsecutiveRegsLast();
11656 }
11657 EVT VT = RetVTs[I];
11658 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11659 unsigned NumRegs =
11660 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11661 for (unsigned i = 0; i != NumRegs; ++i) {
11662 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
11664 if (CLI.RetTy->isPointerTy()) {
11665 Ret.Flags.setPointer();
11667 cast<PointerType>(CLI.RetTy)->getAddressSpace());
11668 }
11669 if (CLI.RetSExt)
11670 Ret.Flags.setSExt();
11671 if (CLI.RetZExt)
11672 Ret.Flags.setZExt();
11673 if (CLI.IsInReg)
11674 Ret.Flags.setInReg();
11675 CLI.Ins.push_back(Ret);
11676 }
11677 }
11678 }
11679
11680 // We push in swifterror return as the last element of CLI.Ins.
11681 ArgListTy &Args = CLI.getArgs();
11682 if (supportSwiftError()) {
11683 for (const ArgListEntry &Arg : Args) {
11684 if (Arg.IsSwiftError) {
11685 ISD::ArgFlagsTy Flags;
11686 Flags.setSwiftError();
11688 PointerType::getUnqual(Context),
11689 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
11690 CLI.Ins.push_back(Ret);
11691 }
11692 }
11693 }
11694
11695 // Handle all of the outgoing arguments.
11696 CLI.Outs.clear();
11697 CLI.OutVals.clear();
11698 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11699 SmallVector<Type *, 4> OrigArgTys;
11700 ComputeValueTypes(DL, Args[i].OrigTy, OrigArgTys);
11701 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11702 Type *FinalType = Args[i].Ty;
11703 if (Args[i].IsByVal)
11704 FinalType = Args[i].IndirectType;
11705 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11706 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
11707 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
11708 ++Value) {
11709 Type *OrigArgTy = OrigArgTys[Value];
11710 Type *ArgTy = OrigArgTy;
11711 if (Args[i].Ty != Args[i].OrigTy) {
11712 assert(Value == 0 && "Only supported for non-aggregate arguments");
11713 ArgTy = Args[i].Ty;
11714 }
11715
11716 EVT VT = getValueType(DL, ArgTy);
11717 SDValue Op = SDValue(Args[i].Node.getNode(),
11718 Args[i].Node.getResNo() + Value);
11719 ISD::ArgFlagsTy Flags;
11720
11721 // Certain targets (such as MIPS), may have a different ABI alignment
11722 // for a type depending on the context. Give the target a chance to
11723 // specify the alignment it wants.
11724 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11725 Flags.setOrigAlign(OriginalAlignment);
11726
11727 if (i >= CLI.NumFixedArgs)
11728 Flags.setVarArg();
11729 if (ArgTy->isPointerTy()) {
11730 Flags.setPointer();
11731 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
11732 }
11733 if (Args[i].IsZExt)
11734 Flags.setZExt();
11735 if (Args[i].IsSExt)
11736 Flags.setSExt();
11737 if (Args[i].IsNoExt)
11738 Flags.setNoExt();
11739 if (Args[i].IsInReg) {
11740 // If we are using vectorcall calling convention, a structure that is
11741 // passed InReg - is surely an HVA
11743 isa<StructType>(FinalType)) {
11744 // The first value of a structure is marked
11745 if (0 == Value)
11746 Flags.setHvaStart();
11747 Flags.setHva();
11748 }
11749 // Set InReg Flag
11750 Flags.setInReg();
11751 }
11752 if (Args[i].IsSRet)
11753 Flags.setSRet();
11754 if (Args[i].IsSwiftSelf)
11755 Flags.setSwiftSelf();
11756 if (Args[i].IsSwiftAsync)
11757 Flags.setSwiftAsync();
11758 if (Args[i].IsSwiftError)
11759 Flags.setSwiftError();
11760 if (Args[i].IsCFGuardTarget)
11761 Flags.setCFGuardTarget();
11762 if (Args[i].IsByVal)
11763 Flags.setByVal();
11764 if (Args[i].IsByRef)
11765 Flags.setByRef();
11766 if (Args[i].IsPreallocated) {
11767 Flags.setPreallocated();
11768 // Set the byval flag for CCAssignFn callbacks that don't know about
11769 // preallocated. This way we can know how many bytes we should've
11770 // allocated and how many bytes a callee cleanup function will pop. If
11771 // we port preallocated to more targets, we'll have to add custom
11772 // preallocated handling in the various CC lowering callbacks.
11773 Flags.setByVal();
11774 }
11775 if (Args[i].IsInAlloca) {
11776 Flags.setInAlloca();
11777 // Set the byval flag for CCAssignFn callbacks that don't know about
11778 // inalloca. This way we can know how many bytes we should've allocated
11779 // and how many bytes a callee cleanup function will pop. If we port
11780 // inalloca to more targets, we'll have to add custom inalloca handling
11781 // in the various CC lowering callbacks.
11782 Flags.setByVal();
11783 }
11784 Align MemAlign;
11785 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11786 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
11787 Flags.setByValSize(FrameSize);
11788
11789 // info is not there but there are cases it cannot get right.
11790 if (auto MA = Args[i].Alignment)
11791 MemAlign = *MA;
11792 else
11793 MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL);
11794 } else if (auto MA = Args[i].Alignment) {
11795 MemAlign = *MA;
11796 } else {
11797 MemAlign = OriginalAlignment;
11798 }
11799 Flags.setMemAlign(MemAlign);
11800 if (Args[i].IsNest)
11801 Flags.setNest();
11802 if (NeedsRegBlock)
11803 Flags.setInConsecutiveRegs();
11804
11805 MVT PartVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11806 unsigned NumParts =
11807 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11810
11811 if (Args[i].IsSExt)
11813 else if (Args[i].IsZExt)
11815
11816 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11817 // for now.
11818 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11820 assert((CLI.RetTy == Args[i].Ty ||
11821 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11823 Args[i].Ty->getPointerAddressSpace())) &&
11824 RetVTs.size() == NumValues && "unexpected use of 'returned'");
11825 // Before passing 'returned' to the target lowering code, ensure that
11826 // either the register MVT and the actual EVT are the same size or that
11827 // the return value and argument are extended in the same way; in these
11828 // cases it's safe to pass the argument register value unchanged as the
11829 // return register value (although it's at the target's option whether
11830 // to do so)
11831 // TODO: allow code generation to take advantage of partially preserved
11832 // registers rather than clobbering the entire register when the
11833 // parameter extension method is not compatible with the return
11834 // extension method
11835 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11836 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11837 CLI.RetZExt == Args[i].IsZExt))
11838 Flags.setReturned();
11839 }
11840
11841 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
11842 CLI.CallConv, ExtendKind);
11843
11844 for (unsigned j = 0; j != NumParts; ++j) {
11845 // if it isn't first piece, alignment must be 1
11846 // For scalable vectors the scalable part is currently handled
11847 // by individual targets, so we just use the known minimum size here.
11848 ISD::OutputArg MyFlags(
11849 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
11850 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11851 if (NumParts > 1 && j == 0)
11852 MyFlags.Flags.setSplit();
11853 else if (j != 0) {
11854 MyFlags.Flags.setOrigAlign(Align(1));
11855 if (j == NumParts - 1)
11856 MyFlags.Flags.setSplitEnd();
11857 }
11858
11859 CLI.Outs.push_back(MyFlags);
11860 CLI.OutVals.push_back(Parts[j]);
11861 }
11862
11863 if (NeedsRegBlock && Value == NumValues - 1)
11864 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11865 }
11866 }
11867
11869 CLI.Chain = LowerCall(CLI, InVals);
11870
11871 // Update CLI.InVals to use outside of this function.
11872 CLI.InVals = InVals;
11873
11874 // Verify that the target's LowerCall behaved as expected.
11875 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11876 "LowerCall didn't return a valid chain!");
11877 assert((!CLI.IsTailCall || InVals.empty()) &&
11878 "LowerCall emitted a return value for a tail call!");
11879 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11880 "LowerCall didn't emit the correct number of values!");
11881
11882 // For a tail call, the return value is merely live-out and there aren't
11883 // any nodes in the DAG representing it. Return a special value to
11884 // indicate that a tail call has been emitted and no more Instructions
11885 // should be processed in the current block.
11886 if (CLI.IsTailCall) {
11887 CLI.DAG.setRoot(CLI.Chain);
11888 return std::make_pair(SDValue(), SDValue());
11889 }
11890
11891#ifndef NDEBUG
11892 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11893 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11894 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11895 "LowerCall emitted a value with the wrong type!");
11896 }
11897#endif
11898
11899 SmallVector<SDValue, 4> ReturnValues;
11900 if (!CanLowerReturn) {
11901 // The instruction result is the result of loading from the
11902 // hidden sret parameter.
11903 MVT PtrVT = getPointerTy(DL, DL.getAllocaAddrSpace());
11904
11905 unsigned NumValues = RetVTs.size();
11906 ReturnValues.resize(NumValues);
11907 SmallVector<SDValue, 4> Chains(NumValues);
11908
11909 // An aggregate return value cannot wrap around the address space, so
11910 // offsets to its parts don't wrap either.
11912 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
11913 for (unsigned i = 0; i < NumValues; ++i) {
11915 DemoteStackSlot, CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT),
11917 SDValue L = CLI.DAG.getLoad(
11918 RetVTs[i], CLI.DL, CLI.Chain, Add,
11920 DemoteStackIdx, Offsets[i]),
11921 HiddenSRetAlign);
11922 ReturnValues[i] = L;
11923 Chains[i] = L.getValue(1);
11924 }
11925
11926 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
11927 } else {
11928 // Collect the legal value parts into potentially illegal values
11929 // that correspond to the original function's return values.
11930 std::optional<ISD::NodeType> AssertOp;
11931 if (CLI.RetSExt)
11932 AssertOp = ISD::AssertSext;
11933 else if (CLI.RetZExt)
11934 AssertOp = ISD::AssertZext;
11935 unsigned CurReg = 0;
11936 for (EVT VT : RetVTs) {
11937 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11938 unsigned NumRegs =
11939 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11940
11941 ReturnValues.push_back(getCopyFromParts(
11942 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
11943 CLI.Chain, CLI.CallConv, AssertOp));
11944 CurReg += NumRegs;
11945 }
11946
11947 // For a function returning void, there is no return value. We can't create
11948 // such a node, so we just return a null return value in that case. In
11949 // that case, nothing will actually look at the value.
11950 if (ReturnValues.empty())
11951 return std::make_pair(SDValue(), CLI.Chain);
11952 }
11953
11954 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
11955 CLI.DAG.getVTList(RetVTs), ReturnValues);
11956 return std::make_pair(Res, CLI.Chain);
11957}
11958
11959/// Places new result values for the node in Results (their number
11960/// and types must exactly match those of the original return values of
11961/// the node), or leaves Results empty, which indicates that the node is not
11962/// to be custom lowered after all.
11965 SelectionDAG &DAG) const {
11966 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
11967
11968 if (!Res.getNode())
11969 return;
11970
11971 // If the original node has one result, take the return value from
11972 // LowerOperation as is. It might not be result number 0.
11973 if (N->getNumValues() == 1) {
11974 Results.push_back(Res);
11975 return;
11976 }
11977
11978 // If the original node has multiple results, then the return node should
11979 // have the same number of results.
11980 assert((N->getNumValues() == Res->getNumValues()) &&
11981 "Lowering returned the wrong number of results!");
11982
11983 // Places new result values base on N result number.
11984 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11985 Results.push_back(Res.getValue(I));
11986}
11987
11989 llvm_unreachable("LowerOperation not implemented for this target!");
11990}
11991
11993 Register Reg,
11994 ISD::NodeType ExtendType) {
11996 assert((Op.getOpcode() != ISD::CopyFromReg ||
11997 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11998 "Copy from a reg to the same reg!");
11999 assert(!Reg.isPhysical() && "Is a physreg");
12000
12001 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12002 // If this is an InlineAsm we have to match the registers required, not the
12003 // notional registers required by the type.
12004
12005 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
12006 std::nullopt); // This is not an ABI copy.
12007 SDValue Chain = DAG.getEntryNode();
12008
12009 if (ExtendType == ISD::ANY_EXTEND) {
12010 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
12011 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
12012 ExtendType = PreferredExtendIt->second;
12013 }
12014 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
12015 PendingExports.push_back(Chain);
12016}
12017
12019
12020/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
12021/// entry block, return true. This includes arguments used by switches, since
12022/// the switch may expand into multiple basic blocks.
12023static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
12024 // With FastISel active, we may be splitting blocks, so force creation
12025 // of virtual registers for all non-dead arguments.
12026 if (FastISel)
12027 return A->use_empty();
12028
12029 const BasicBlock &Entry = A->getParent()->front();
12030 for (const User *U : A->users())
12031 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
12032 return false; // Use not in entry block.
12033
12034 return true;
12035}
12036
12038 DenseMap<const Argument *,
12039 std::pair<const AllocaInst *, const StoreInst *>>;
12040
12041/// Scan the entry block of the function in FuncInfo for arguments that look
12042/// like copies into a local alloca. Record any copied arguments in
12043/// ArgCopyElisionCandidates.
12044static void
12046 FunctionLoweringInfo *FuncInfo,
12047 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
12048 // Record the state of every static alloca used in the entry block. Argument
12049 // allocas are all used in the entry block, so we need approximately as many
12050 // entries as we have arguments.
12051 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
12053 unsigned NumArgs = FuncInfo->Fn->arg_size();
12054 StaticAllocas.reserve(NumArgs * 2);
12055
12056 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
12057 if (!V)
12058 return nullptr;
12059 V = V->stripPointerCasts();
12060 const auto *AI = dyn_cast<AllocaInst>(V);
12061 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
12062 return nullptr;
12063 auto Iter = StaticAllocas.insert({AI, Unknown});
12064 return &Iter.first->second;
12065 };
12066
12067 // Look for stores of arguments to static allocas. Look through bitcasts and
12068 // GEPs to handle type coercions, as long as the alloca is fully initialized
12069 // by the store. Any non-store use of an alloca escapes it and any subsequent
12070 // unanalyzed store might write it.
12071 // FIXME: Handle structs initialized with multiple stores.
12072 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
12073 // Look for stores, and handle non-store uses conservatively.
12074 const auto *SI = dyn_cast<StoreInst>(&I);
12075 if (!SI) {
12076 // We will look through cast uses, so ignore them completely.
12077 if (I.isCast())
12078 continue;
12079 // Ignore pseudo op intrinsics, they don't escape or store to allocas.
12081 continue;
12082 // This is an unknown instruction. Assume it escapes or writes to all
12083 // static alloca operands.
12084 for (const Use &U : I.operands()) {
12085 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
12086 *Info = StaticAllocaInfo::Clobbered;
12087 }
12088 continue;
12089 }
12090
12091 // If the stored value is a static alloca, mark it as escaped.
12092 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
12093 *Info = StaticAllocaInfo::Clobbered;
12094
12095 // Check if the destination is a static alloca.
12096 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
12097 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
12098 if (!Info)
12099 continue;
12100 const AllocaInst *AI = cast<AllocaInst>(Dst);
12101
12102 // Skip allocas that have been initialized or clobbered.
12103 if (*Info != StaticAllocaInfo::Unknown)
12104 continue;
12105
12106 // Check if the stored value is an argument, and that this store fully
12107 // initializes the alloca.
12108 // If the argument type has padding bits we can't directly forward a pointer
12109 // as the upper bits may contain garbage.
12110 // Don't elide copies from the same argument twice.
12111 const Value *Val = SI->getValueOperand()->stripPointerCasts();
12112 const auto *Arg = dyn_cast<Argument>(Val);
12113 std::optional<TypeSize> AllocaSize = AI->getAllocationSize(DL);
12114 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
12115 Arg->getType()->isEmptyTy() || !AllocaSize ||
12116 DL.getTypeStoreSize(Arg->getType()) != *AllocaSize ||
12117 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
12118 ArgCopyElisionCandidates.count(Arg)) {
12119 *Info = StaticAllocaInfo::Clobbered;
12120 continue;
12121 }
12122
12123 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
12124 << '\n');
12125
12126 // Mark this alloca and store for argument copy elision.
12127 *Info = StaticAllocaInfo::Elidable;
12128 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
12129
12130 // Stop scanning if we've seen all arguments. This will happen early in -O0
12131 // builds, which is useful, because -O0 builds have large entry blocks and
12132 // many allocas.
12133 if (ArgCopyElisionCandidates.size() == NumArgs)
12134 break;
12135 }
12136}
12137
12138/// Try to elide argument copies from memory into a local alloca. Succeeds if
12139/// ArgVal is a load from a suitable fixed stack object.
12142 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
12143 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
12144 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
12145 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
12146 // Check if this is a load from a fixed stack object.
12147 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
12148 if (!LNode)
12149 return;
12150 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
12151 if (!FINode)
12152 return;
12153
12154 // Check that the fixed stack object is the right size and alignment.
12155 // Look at the alignment that the user wrote on the alloca instead of looking
12156 // at the stack object.
12157 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
12158 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
12159 const AllocaInst *AI = ArgCopyIter->second.first;
12160 int FixedIndex = FINode->getIndex();
12161 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
12162 int OldIndex = AllocaIndex;
12163 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
12164 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
12165 LLVM_DEBUG(
12166 dbgs() << " argument copy elision failed due to bad fixed stack "
12167 "object size\n");
12168 return;
12169 }
12170 Align RequiredAlignment = AI->getAlign();
12171 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
12172 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
12173 "greater than stack argument alignment ("
12174 << DebugStr(RequiredAlignment) << " vs "
12175 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
12176 return;
12177 }
12178
12179 // Perform the elision. Delete the old stack object and replace its only use
12180 // in the variable info map. Mark the stack object as mutable and aliased.
12181 LLVM_DEBUG({
12182 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
12183 << " Replacing frame index " << OldIndex << " with " << FixedIndex
12184 << '\n';
12185 });
12186 MFI.RemoveStackObject(OldIndex);
12187 MFI.setIsImmutableObjectIndex(FixedIndex, false);
12188 MFI.setIsAliasedObjectIndex(FixedIndex, true);
12189 AllocaIndex = FixedIndex;
12190 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
12191 for (SDValue ArgVal : ArgVals)
12192 Chains.push_back(ArgVal.getValue(1));
12193
12194 // Avoid emitting code for the store implementing the copy.
12195 const StoreInst *SI = ArgCopyIter->second.second;
12196 ElidedArgCopyInstrs.insert(SI);
12197
12198 // Check for uses of the argument again so that we can avoid exporting ArgVal
12199 // if it is't used by anything other than the store.
12200 for (const Value *U : Arg.users()) {
12201 if (U != SI) {
12202 ArgHasUses = true;
12203 break;
12204 }
12205 }
12206}
12207
12208void SelectionDAGISel::LowerArguments(const Function &F, bool FastISel) {
12209 SelectionDAG &DAG = SDB->DAG;
12210 SDLoc dl = SDB->getCurSDLoc();
12211 const DataLayout &DL = DAG.getDataLayout();
12213
12214 // In Naked functions we aren't going to save any registers.
12215 if (F.hasFnAttribute(Attribute::Naked))
12216 return;
12217
12218 if (!FuncInfo->CanLowerReturn) {
12219 // Put in an sret pointer parameter before all the other parameters.
12220 MVT ValueVT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
12221
12222 ISD::ArgFlagsTy Flags;
12223 Flags.setSRet();
12224 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVT);
12225 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
12227 Ins.push_back(RetArg);
12228 }
12229
12230 // Look for stores of arguments to static allocas. Mark such arguments with a
12231 // flag to ask the target to give us the memory location of that argument if
12232 // available.
12233 ArgCopyElisionMapTy ArgCopyElisionCandidates;
12235 ArgCopyElisionCandidates);
12236
12237 // Set up the incoming argument description vector.
12238 for (const Argument &Arg : F.args()) {
12239 unsigned ArgNo = Arg.getArgNo();
12241 ComputeValueTypes(DAG.getDataLayout(), Arg.getType(), Types);
12242 bool isArgValueUsed = !Arg.use_empty();
12243 Type *FinalType = Arg.getType();
12244 if (Arg.hasAttribute(Attribute::ByVal))
12245 FinalType = Arg.getParamByValType();
12246 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
12247 FinalType, F.getCallingConv(), F.isVarArg(), DL);
12248 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
12249 ++Value) {
12250 Type *ArgTy = Types[Value];
12251 EVT VT = TLI->getValueType(DL, ArgTy);
12252 ISD::ArgFlagsTy Flags;
12253
12254 if (ArgTy->isPointerTy()) {
12255 Flags.setPointer();
12256 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
12257 }
12258 if (Arg.hasAttribute(Attribute::ZExt))
12259 Flags.setZExt();
12260 if (Arg.hasAttribute(Attribute::SExt))
12261 Flags.setSExt();
12262 if (Arg.hasAttribute(Attribute::InReg)) {
12263 // If we are using vectorcall calling convention, a structure that is
12264 // passed InReg - is surely an HVA
12265 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
12266 isa<StructType>(Arg.getType())) {
12267 // The first value of a structure is marked
12268 if (0 == Value)
12269 Flags.setHvaStart();
12270 Flags.setHva();
12271 }
12272 // Set InReg Flag
12273 Flags.setInReg();
12274 }
12275 if (Arg.hasAttribute(Attribute::StructRet))
12276 Flags.setSRet();
12277 if (Arg.hasAttribute(Attribute::SwiftSelf))
12278 Flags.setSwiftSelf();
12279 if (Arg.hasAttribute(Attribute::SwiftAsync))
12280 Flags.setSwiftAsync();
12281 if (Arg.hasAttribute(Attribute::SwiftError))
12282 Flags.setSwiftError();
12283 if (Arg.hasAttribute(Attribute::ByVal))
12284 Flags.setByVal();
12285 if (Arg.hasAttribute(Attribute::ByRef))
12286 Flags.setByRef();
12287 if (Arg.hasAttribute(Attribute::InAlloca)) {
12288 Flags.setInAlloca();
12289 // Set the byval flag for CCAssignFn callbacks that don't know about
12290 // inalloca. This way we can know how many bytes we should've allocated
12291 // and how many bytes a callee cleanup function will pop. If we port
12292 // inalloca to more targets, we'll have to add custom inalloca handling
12293 // in the various CC lowering callbacks.
12294 Flags.setByVal();
12295 }
12296 if (Arg.hasAttribute(Attribute::Preallocated)) {
12297 Flags.setPreallocated();
12298 // Set the byval flag for CCAssignFn callbacks that don't know about
12299 // preallocated. This way we can know how many bytes we should've
12300 // allocated and how many bytes a callee cleanup function will pop. If
12301 // we port preallocated to more targets, we'll have to add custom
12302 // preallocated handling in the various CC lowering callbacks.
12303 Flags.setByVal();
12304 }
12305
12306 // Certain targets (such as MIPS), may have a different ABI alignment
12307 // for a type depending on the context. Give the target a chance to
12308 // specify the alignment it wants.
12309 const Align OriginalAlignment(
12310 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
12311 Flags.setOrigAlign(OriginalAlignment);
12312
12313 Align MemAlign;
12314 Type *ArgMemTy = nullptr;
12315 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
12316 Flags.isByRef()) {
12317 if (!ArgMemTy)
12318 ArgMemTy = Arg.getPointeeInMemoryValueType();
12319
12320 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
12321
12322 // For in-memory arguments, size and alignment should be passed from FE.
12323 // BE will guess if this info is not there but there are cases it cannot
12324 // get right.
12325 if (auto ParamAlign = Arg.getParamStackAlign())
12326 MemAlign = *ParamAlign;
12327 else if ((ParamAlign = Arg.getParamAlign()))
12328 MemAlign = *ParamAlign;
12329 else
12330 MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL);
12331 if (Flags.isByRef())
12332 Flags.setByRefSize(MemSize);
12333 else
12334 Flags.setByValSize(MemSize);
12335 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
12336 MemAlign = *ParamAlign;
12337 } else {
12338 MemAlign = OriginalAlignment;
12339 }
12340 Flags.setMemAlign(MemAlign);
12341
12342 if (Arg.hasAttribute(Attribute::Nest))
12343 Flags.setNest();
12344 if (NeedsRegBlock)
12345 Flags.setInConsecutiveRegs();
12346 if (ArgCopyElisionCandidates.count(&Arg))
12347 Flags.setCopyElisionCandidate();
12348 if (Arg.hasAttribute(Attribute::Returned))
12349 Flags.setReturned();
12350
12351 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
12352 *CurDAG->getContext(), F.getCallingConv(), VT);
12353 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
12354 *CurDAG->getContext(), F.getCallingConv(), VT);
12355 for (unsigned i = 0; i != NumRegs; ++i) {
12356 // For scalable vectors, use the minimum size; individual targets
12357 // are responsible for handling scalable vector arguments and
12358 // return values.
12359 ISD::InputArg MyFlags(
12360 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12361 i * RegisterVT.getStoreSize().getKnownMinValue());
12362 if (NumRegs > 1 && i == 0)
12363 MyFlags.Flags.setSplit();
12364 // if it isn't first piece, alignment must be 1
12365 else if (i > 0) {
12366 MyFlags.Flags.setOrigAlign(Align(1));
12367 if (i == NumRegs - 1)
12368 MyFlags.Flags.setSplitEnd();
12369 }
12370 Ins.push_back(MyFlags);
12371 }
12372 if (NeedsRegBlock && Value == NumValues - 1)
12373 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
12374 }
12375 }
12376
12377 // Call the target to set up the argument values.
12379 SDValue NewRoot = TLI->LowerFormalArguments(
12380 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
12381
12382 // Verify that the target's LowerFormalArguments behaved as expected.
12383 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
12384 "LowerFormalArguments didn't return a valid chain!");
12385 assert(InVals.size() == Ins.size() &&
12386 "LowerFormalArguments didn't emit the correct number of values!");
12387 assert(all_of(InVals, [](SDValue InVal) { return InVal.getNode(); }) &&
12388 "LowerFormalArguments emitted a null value!");
12389
12390 // Update the DAG with the new chain value resulting from argument lowering.
12391 DAG.setRoot(NewRoot);
12392
12393 // Set up the argument values.
12394 unsigned i = 0;
12395 if (!FuncInfo->CanLowerReturn) {
12396 // Create a virtual register for the sret pointer, and put in a copy
12397 // from the sret argument into it.
12398 MVT VT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
12399 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
12400 std::optional<ISD::NodeType> AssertOp;
12401 SDValue ArgValue =
12402 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
12403 F.getCallingConv(), AssertOp);
12404
12405 MachineFunction& MF = SDB->DAG.getMachineFunction();
12406 MachineRegisterInfo& RegInfo = MF.getRegInfo();
12407 Register SRetReg =
12408 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
12409 FuncInfo->DemoteRegister = SRetReg;
12410 NewRoot =
12411 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
12412 DAG.setRoot(NewRoot);
12413
12414 // i indexes lowered arguments. Bump it past the hidden sret argument.
12415 ++i;
12416 }
12417
12419 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12420 for (const Argument &Arg : F.args()) {
12421 SmallVector<SDValue, 4> ArgValues;
12422 SmallVector<EVT, 4> ValueVTs;
12423 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
12424 unsigned NumValues = ValueVTs.size();
12425 if (NumValues == 0)
12426 continue;
12427
12428 bool ArgHasUses = !Arg.use_empty();
12429
12430 // Elide the copying store if the target loaded this argument from a
12431 // suitable fixed stack object.
12432 if (Ins[i].Flags.isCopyElisionCandidate()) {
12433 unsigned NumParts = 0;
12434 for (EVT VT : ValueVTs)
12435 NumParts += TLI->getNumRegistersForCallingConv(*CurDAG->getContext(),
12436 F.getCallingConv(), VT);
12437
12438 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
12439 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
12440 ArrayRef(&InVals[i], NumParts), ArgHasUses);
12441 }
12442
12443 // If this argument is unused then remember its value. It is used to generate
12444 // debugging information.
12445 bool isSwiftErrorArg =
12446 TLI->supportSwiftError() &&
12447 Arg.hasAttribute(Attribute::SwiftError);
12448 if (!ArgHasUses && !isSwiftErrorArg) {
12449 SDB->setUnusedArgValue(&Arg, InVals[i]);
12450
12451 // Also remember any frame index for use in FastISel.
12452 if (FrameIndexSDNode *FI =
12454 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12455 }
12456
12457 for (unsigned Val = 0; Val != NumValues; ++Val) {
12458 EVT VT = ValueVTs[Val];
12459 MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
12460 F.getCallingConv(), VT);
12461 unsigned NumParts = TLI->getNumRegistersForCallingConv(
12462 *CurDAG->getContext(), F.getCallingConv(), VT);
12463
12464 // Even an apparent 'unused' swifterror argument needs to be returned. So
12465 // we do generate a copy for it that can be used on return from the
12466 // function.
12467 if (ArgHasUses || isSwiftErrorArg) {
12468 std::optional<ISD::NodeType> AssertOp;
12469 if (Arg.hasAttribute(Attribute::SExt))
12470 AssertOp = ISD::AssertSext;
12471 else if (Arg.hasAttribute(Attribute::ZExt))
12472 AssertOp = ISD::AssertZext;
12473
12474 SDValue OutVal =
12475 getCopyFromParts(DAG, dl, &InVals[i], NumParts, PartVT, VT, nullptr,
12476 NewRoot, F.getCallingConv(), AssertOp);
12477
12478 FPClassTest NoFPClass = Arg.getNoFPClass();
12479 if (NoFPClass != fcNone) {
12480 SDValue SDNoFPClass = DAG.getTargetConstant(
12481 static_cast<uint64_t>(NoFPClass), dl, MVT::i32);
12482 OutVal = DAG.getNode(ISD::AssertNoFPClass, dl, OutVal.getValueType(),
12483 OutVal, SDNoFPClass);
12484 }
12485 ArgValues.push_back(OutVal);
12486 }
12487
12488 i += NumParts;
12489 }
12490
12491 // We don't need to do anything else for unused arguments.
12492 if (ArgValues.empty())
12493 continue;
12494
12495 // Note down frame index.
12496 if (FrameIndexSDNode *FI =
12497 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
12498 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12499
12500 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
12501 SDB->getCurSDLoc());
12502
12503 SDB->setValue(&Arg, Res);
12504 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
12505 // We want to associate the argument with the frame index, among
12506 // involved operands, that correspond to the lowest address. The
12507 // getCopyFromParts function, called earlier, is swapping the order of
12508 // the operands to BUILD_PAIR depending on endianness. The result of
12509 // that swapping is that the least significant bits of the argument will
12510 // be in the first operand of the BUILD_PAIR node, and the most
12511 // significant bits will be in the second operand.
12512 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
12513 if (LoadSDNode *LNode =
12514 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
12515 if (FrameIndexSDNode *FI =
12516 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
12517 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12518 }
12519
12520 // Analyses past this point are naive and don't expect an assertion.
12521 if (Res.getOpcode() == ISD::AssertZext)
12522 Res = Res.getOperand(0);
12523
12524 // Update the SwiftErrorVRegDefMap.
12525 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
12526 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
12527 if (Reg.isVirtual())
12528 SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
12529 Reg);
12530 }
12531
12532 // If this argument is live outside of the entry block, insert a copy from
12533 // wherever we got it to the vreg that other BB's will reference it as.
12534 if (Res.getOpcode() == ISD::CopyFromReg) {
12535 // If we can, though, try to skip creating an unnecessary vreg.
12536 // FIXME: This isn't very clean... it would be nice to make this more
12537 // general.
12538 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
12539 if (Reg.isVirtual()) {
12540 FuncInfo->ValueMap[&Arg] = Reg;
12541 continue;
12542 }
12543 }
12544 if (!isOnlyUsedInEntryBlock(&Arg, FastISel)) {
12545 FuncInfo->InitializeRegForValue(&Arg);
12546 SDB->CopyToExportRegsIfNeeded(&Arg);
12547 }
12548 }
12549
12550 if (!Chains.empty()) {
12551 Chains.push_back(NewRoot);
12552 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
12553 }
12554
12555 DAG.setRoot(NewRoot);
12556
12557 assert(i == InVals.size() && "Argument register count mismatch!");
12558
12559 // If any argument copy elisions occurred and we have debug info, update the
12560 // stale frame indices used in the dbg.declare variable info table.
12561 if (!ArgCopyElisionFrameIndexMap.empty()) {
12562 for (MachineFunction::VariableDbgInfo &VI :
12563 MF->getInStackSlotVariableDbgInfo()) {
12564 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
12565 if (I != ArgCopyElisionFrameIndexMap.end())
12566 VI.updateStackSlot(I->second);
12567 }
12568 }
12569
12570 // Finally, if the target has anything special to do, allow it to do so.
12572}
12573
12574/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
12575/// ensure constants are generated when needed. Remember the virtual registers
12576/// that need to be added to the Machine PHI nodes as input. We cannot just
12577/// directly add them, because expansion might result in multiple MBB's for one
12578/// BB. As such, the start of the BB might correspond to a different MBB than
12579/// the end.
12580void
12581SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
12582 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12583
12584 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12585
12586 // Check PHI nodes in successors that expect a value to be available from this
12587 // block.
12588 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
12589 if (!isa<PHINode>(SuccBB->begin())) continue;
12590 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB);
12591
12592 // If this terminator has multiple identical successors (common for
12593 // switches), only handle each succ once.
12594 if (!SuccsHandled.insert(SuccMBB).second)
12595 continue;
12596
12598
12599 // At this point we know that there is a 1-1 correspondence between LLVM PHI
12600 // nodes and Machine PHI nodes, but the incoming operands have not been
12601 // emitted yet.
12602 for (const PHINode &PN : SuccBB->phis()) {
12603 // Ignore dead phi's.
12604 if (PN.use_empty())
12605 continue;
12606
12607 // Skip empty types
12608 if (PN.getType()->isEmptyTy())
12609 continue;
12610
12611 Register Reg;
12612 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
12613
12614 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
12615 Register &RegOut = ConstantsOut[C];
12616 if (!RegOut) {
12617 RegOut = FuncInfo.CreateRegs(PHIOp);
12618 // We need to zero/sign extend ConstantInt phi operands to match
12619 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12620 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12621 if (auto *CI = dyn_cast<ConstantInt>(C))
12622 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
12624 CopyValueToVirtualRegister(C, RegOut, ExtendType);
12625 }
12626 Reg = RegOut;
12627 } else {
12628 auto I = FuncInfo.ValueMap.find(PHIOp);
12629 if (I != FuncInfo.ValueMap.end())
12630 Reg = I->second;
12631 else {
12632 assert(isa<AllocaInst>(PHIOp) &&
12633 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12634 "Didn't codegen value into a register!??");
12635 Reg = FuncInfo.CreateRegs(PHIOp);
12637 }
12638 }
12639
12640 // Remember that this register needs to added to the machine PHI node as
12641 // the input for this MBB.
12642 SmallVector<EVT, 4> ValueVTs;
12643 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
12644 for (EVT VT : ValueVTs) {
12645 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
12646 for (unsigned i = 0; i != NumRegisters; ++i)
12647 FuncInfo.PHINodesToUpdate.emplace_back(&*MBBI++, Reg + i);
12648 Reg += NumRegisters;
12649 }
12650 }
12651 }
12652
12653 ConstantsOut.clear();
12654}
12655
12656MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12658 if (++I == FuncInfo.MF->end())
12659 return nullptr;
12660 return &*I;
12661}
12662
12663/// During lowering new call nodes can be created (such as memset, etc.).
12664/// Those will become new roots of the current DAG, but complications arise
12665/// when they are tail calls. In such cases, the call lowering will update
12666/// the root, but the builder still needs to know that a tail call has been
12667/// lowered in order to avoid generating an additional return.
12668void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12669 // If the node is null, we do have a tail call.
12670 if (MaybeTC.getNode() != nullptr)
12671 DAG.setRoot(MaybeTC);
12672 else
12673 HasTailCall = true;
12674}
12675
12676void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12677 MachineBasicBlock *SwitchMBB,
12678 MachineBasicBlock *DefaultMBB) {
12679 MachineFunction *CurMF = FuncInfo.MF;
12680 MachineBasicBlock *NextMBB = nullptr;
12682 if (++BBI != FuncInfo.MF->end())
12683 NextMBB = &*BBI;
12684
12685 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12686
12687 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12688
12689 if (Size == 2 && W.MBB == SwitchMBB) {
12690 // If any two of the cases has the same destination, and if one value
12691 // is the same as the other, but has one bit unset that the other has set,
12692 // use bit manipulation to do two compares at once. For example:
12693 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12694 // TODO: This could be extended to merge any 2 cases in switches with 3
12695 // cases.
12696 // TODO: Handle cases where W.CaseBB != SwitchBB.
12697 CaseCluster &Small = *W.FirstCluster;
12698 CaseCluster &Big = *W.LastCluster;
12699
12700 if (Small.Low == Small.High && Big.Low == Big.High &&
12701 Small.MBB == Big.MBB) {
12702 const APInt &SmallValue = Small.Low->getValue();
12703 const APInt &BigValue = Big.Low->getValue();
12704
12705 // Check that there is only one bit different.
12706 APInt CommonBit = BigValue ^ SmallValue;
12707 if (CommonBit.isPowerOf2()) {
12708 SDValue CondLHS = getValue(Cond);
12709 EVT VT = CondLHS.getValueType();
12710 SDLoc DL = getCurSDLoc();
12711
12712 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
12713 DAG.getConstant(CommonBit, DL, VT));
12714 SDValue Cond = DAG.getSetCC(
12715 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
12716 ISD::SETEQ);
12717
12718 // Update successor info.
12719 // Both Small and Big will jump to Small.BB, so we sum up the
12720 // probabilities.
12721 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
12722 if (BPI)
12723 addSuccessorWithProb(
12724 SwitchMBB, DefaultMBB,
12725 // The default destination is the first successor in IR.
12726 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
12727 else
12728 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12729
12730 // Insert the true branch.
12731 SDValue BrCond =
12732 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
12733 DAG.getBasicBlock(Small.MBB));
12734 // Insert the false branch.
12735 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
12736 DAG.getBasicBlock(DefaultMBB));
12737
12738 DAG.setRoot(BrCond);
12739 return;
12740 }
12741 }
12742 }
12743
12744 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12745 // Here, we order cases by probability so the most likely case will be
12746 // checked first. However, two clusters can have the same probability in
12747 // which case their relative ordering is non-deterministic. So we use Low
12748 // as a tie-breaker as clusters are guaranteed to never overlap.
12749 llvm::sort(W.FirstCluster, W.LastCluster + 1,
12750 [](const CaseCluster &a, const CaseCluster &b) {
12751 return a.Prob != b.Prob ?
12752 a.Prob > b.Prob :
12753 a.Low->getValue().slt(b.Low->getValue());
12754 });
12755
12756 // Rearrange the case blocks so that the last one falls through if possible
12757 // without changing the order of probabilities.
12758 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12759 --I;
12760 if (I->Prob > W.LastCluster->Prob)
12761 break;
12762 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12763 std::swap(*I, *W.LastCluster);
12764 break;
12765 }
12766 }
12767 }
12768
12769 // Compute total probability.
12770 BranchProbability DefaultProb = W.DefaultProb;
12771 BranchProbability UnhandledProbs = DefaultProb;
12772 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12773 UnhandledProbs += I->Prob;
12774
12775 MachineBasicBlock *CurMBB = W.MBB;
12776 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12777 bool FallthroughUnreachable = false;
12778 MachineBasicBlock *Fallthrough;
12779 if (I == W.LastCluster) {
12780 // For the last cluster, fall through to the default destination.
12781 Fallthrough = DefaultMBB;
12782 FallthroughUnreachable =
12784 } else {
12785 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
12786 CurMF->insert(BBI, Fallthrough);
12787 // Put Cond in a virtual register to make it available from the new blocks.
12789 }
12790 UnhandledProbs -= I->Prob;
12791
12792 switch (I->Kind) {
12793 case CC_JumpTable: {
12794 // FIXME: Optimize away range check based on pivot comparisons.
12795 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12796 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12797
12798 // The jump block hasn't been inserted yet; insert it here.
12799 MachineBasicBlock *JumpMBB = JT->MBB;
12800 CurMF->insert(BBI, JumpMBB);
12801
12802 auto JumpProb = I->Prob;
12803 auto FallthroughProb = UnhandledProbs;
12804
12805 // If the default statement is a target of the jump table, we evenly
12806 // distribute the default probability to successors of CurMBB. Also
12807 // update the probability on the edge from JumpMBB to Fallthrough.
12808 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12809 SE = JumpMBB->succ_end();
12810 SI != SE; ++SI) {
12811 if (*SI == DefaultMBB) {
12812 JumpProb += DefaultProb / 2;
12813 FallthroughProb -= DefaultProb / 2;
12814 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
12815 JumpMBB->normalizeSuccProbs();
12816 break;
12817 }
12818 }
12819
12820 // If the default clause is unreachable, propagate that knowledge into
12821 // JTH->FallthroughUnreachable which will use it to suppress the range
12822 // check.
12823 //
12824 // However, don't do this if we're doing branch target enforcement
12825 // (e.g. Arm BTI, or the cf-protection-branch module flag used by
12826 // RISC-V Zicfilp and X86 IBT), because a table branch _without_ a
12827 // range check can be a tempting JOP gadget - out-of-bounds inputs
12828 // that are impossible in correct execution become possible again if
12829 // an attacker can influence the control flow. So if an attacker
12830 // doesn't already have a branch target enforcement bypass available,
12831 // we don't want them to be able to get one out of this table branch.
12832 if (FallthroughUnreachable) {
12833 Function &CurFunc = CurMF->getFunction();
12834 bool HasCFBranch = false;
12835 if (auto *CFBranch = mdconst::extract_or_null<ConstantInt>(
12836 CurFunc.getParent()->getModuleFlag("cf-protection-branch")))
12837 HasCFBranch = !CFBranch->isZero();
12838 if (!HasCFBranch &&
12839 !CurFunc.hasFnAttribute("branch-target-enforcement"))
12840 JTH->FallthroughUnreachable = true;
12841 }
12842
12843 if (!JTH->FallthroughUnreachable)
12844 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12845 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12846 CurMBB->normalizeSuccProbs();
12847
12848 // The jump table header will be inserted in our current block, do the
12849 // range check, and fall through to our fallthrough block.
12850 JTH->HeaderBB = CurMBB;
12851 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12852
12853 // If we're in the right place, emit the jump table header right now.
12854 if (CurMBB == SwitchMBB) {
12855 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
12856 JTH->Emitted = true;
12857 }
12858 break;
12859 }
12860 case CC_BitTests: {
12861 // FIXME: Optimize away range check based on pivot comparisons.
12862 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12863
12864 // The bit test blocks haven't been inserted yet; insert them here.
12865 for (BitTestCase &BTC : BTB->Cases)
12866 CurMF->insert(BBI, BTC.ThisBB);
12867
12868 // Fill in fields of the BitTestBlock.
12869 BTB->Parent = CurMBB;
12870 BTB->Default = Fallthrough;
12871
12872 BTB->DefaultProb = UnhandledProbs;
12873 // If the cases in bit test don't form a contiguous range, we evenly
12874 // distribute the probability on the edge to Fallthrough to two
12875 // successors of CurMBB.
12876 if (!BTB->ContiguousRange) {
12877 BTB->Prob += DefaultProb / 2;
12878 BTB->DefaultProb -= DefaultProb / 2;
12879 }
12880
12881 if (FallthroughUnreachable)
12882 BTB->FallthroughUnreachable = true;
12883
12884 // If we're in the right place, emit the bit test header right now.
12885 if (CurMBB == SwitchMBB) {
12886 visitBitTestHeader(*BTB, SwitchMBB);
12887 BTB->Emitted = true;
12888 }
12889 break;
12890 }
12891 case CC_Range: {
12892 const Value *RHS, *LHS, *MHS;
12894 if (I->Low == I->High) {
12895 // Check Cond == I->Low.
12896 CC = ISD::SETEQ;
12897 LHS = Cond;
12898 RHS=I->Low;
12899 MHS = nullptr;
12900 } else {
12901 // Check I->Low <= Cond <= I->High.
12902 CC = ISD::SETLE;
12903 LHS = I->Low;
12904 MHS = Cond;
12905 RHS = I->High;
12906 }
12907
12908 // If Fallthrough is unreachable, fold away the comparison.
12909 if (FallthroughUnreachable)
12910 CC = ISD::SETTRUE;
12911
12912 // The false probability is the sum of all unhandled cases.
12913 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12914 getCurSDLoc(), I->Prob, UnhandledProbs);
12915
12916 if (CurMBB == SwitchMBB)
12917 visitSwitchCase(CB, SwitchMBB);
12918 else
12919 SL->SwitchCases.push_back(CB);
12920
12921 break;
12922 }
12923 }
12924 CurMBB = Fallthrough;
12925 }
12926}
12927
12928void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12929 const SwitchWorkListItem &W,
12930 Value *Cond,
12931 MachineBasicBlock *SwitchMBB) {
12932 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12933 "Clusters not sorted?");
12934 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12935
12936 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12937 SL->computeSplitWorkItemInfo(W);
12938
12939 // Use the first element on the right as pivot since we will make less-than
12940 // comparisons against it.
12941 CaseClusterIt PivotCluster = FirstRight;
12942 assert(PivotCluster > W.FirstCluster);
12943 assert(PivotCluster <= W.LastCluster);
12944
12945 CaseClusterIt FirstLeft = W.FirstCluster;
12946 CaseClusterIt LastRight = W.LastCluster;
12947
12948 const ConstantInt *Pivot = PivotCluster->Low;
12949
12950 // New blocks will be inserted immediately after the current one.
12952 ++BBI;
12953
12954 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12955 // we can branch to its destination directly if it's squeezed exactly in
12956 // between the known lower bound and Pivot - 1.
12957 MachineBasicBlock *LeftMBB;
12958 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12959 FirstLeft->Low == W.GE &&
12960 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12961 LeftMBB = FirstLeft->MBB;
12962 } else {
12963 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12964 FuncInfo.MF->insert(BBI, LeftMBB);
12965 WorkList.push_back(
12966 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
12967 // Put Cond in a virtual register to make it available from the new blocks.
12969 }
12970
12971 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12972 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12973 // directly if RHS.High equals the current upper bound.
12974 MachineBasicBlock *RightMBB;
12975 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12976 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12977 RightMBB = FirstRight->MBB;
12978 } else {
12979 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12980 FuncInfo.MF->insert(BBI, RightMBB);
12981 WorkList.push_back(
12982 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
12983 // Put Cond in a virtual register to make it available from the new blocks.
12985 }
12986
12987 // Create the CaseBlock record that will be used to lower the branch.
12988 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12989 getCurSDLoc(), LeftProb, RightProb);
12990
12991 if (W.MBB == SwitchMBB)
12992 visitSwitchCase(CB, SwitchMBB);
12993 else
12994 SL->SwitchCases.push_back(CB);
12995}
12996
12997// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12998// from the swith statement.
13000 BranchProbability PeeledCaseProb) {
13001 if (PeeledCaseProb == BranchProbability::getOne())
13003 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
13004
13005 uint32_t Numerator = CaseProb.getNumerator();
13006 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
13007 return BranchProbability(Numerator, std::max(Numerator, Denominator));
13008}
13009
13010// Try to peel the top probability case if it exceeds the threshold.
13011// Return current MachineBasicBlock for the switch statement if the peeling
13012// does not occur.
13013// If the peeling is performed, return the newly created MachineBasicBlock
13014// for the peeled switch statement. Also update Clusters to remove the peeled
13015// case. PeeledCaseProb is the BranchProbability for the peeled case.
13016MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
13017 const SwitchInst &SI, CaseClusterVector &Clusters,
13018 BranchProbability &PeeledCaseProb) {
13019 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
13020 // Don't perform if there is only one cluster or optimizing for size.
13021 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
13022 TM.getOptLevel() == CodeGenOptLevel::None ||
13023 SwitchMBB->getParent()->getFunction().hasMinSize())
13024 return SwitchMBB;
13025
13026 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
13027 unsigned PeeledCaseIndex = 0;
13028 bool SwitchPeeled = false;
13029 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
13030 CaseCluster &CC = Clusters[Index];
13031 if (CC.Prob < TopCaseProb)
13032 continue;
13033 TopCaseProb = CC.Prob;
13034 PeeledCaseIndex = Index;
13035 SwitchPeeled = true;
13036 }
13037 if (!SwitchPeeled)
13038 return SwitchMBB;
13039
13040 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
13041 << TopCaseProb << "\n");
13042
13043 // Record the MBB for the peeled switch statement.
13044 MachineFunction::iterator BBI(SwitchMBB);
13045 ++BBI;
13046 MachineBasicBlock *PeeledSwitchMBB =
13047 FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
13048 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
13049
13050 ExportFromCurrentBlock(SI.getCondition());
13051 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
13052 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
13053 nullptr, nullptr, TopCaseProb.getCompl()};
13054 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
13055
13056 Clusters.erase(PeeledCaseIt);
13057 for (CaseCluster &CC : Clusters) {
13058 LLVM_DEBUG(
13059 dbgs() << "Scale the probablity for one cluster, before scaling: "
13060 << CC.Prob << "\n");
13061 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
13062 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
13063 }
13064 PeeledCaseProb = TopCaseProb;
13065 return PeeledSwitchMBB;
13066}
13067
13068void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
13069 // Extract cases from the switch.
13070 BranchProbabilityInfo *BPI = FuncInfo.BPI;
13071 CaseClusterVector Clusters;
13072 Clusters.reserve(SI.getNumCases());
13073 for (auto I : SI.cases()) {
13074 MachineBasicBlock *Succ = FuncInfo.getMBB(I.getCaseSuccessor());
13075 const ConstantInt *CaseVal = I.getCaseValue();
13076 BranchProbability Prob =
13077 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
13078 : BranchProbability(1, SI.getNumCases() + 1);
13079 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
13080 }
13081
13082 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(SI.getDefaultDest());
13083
13084 // Cluster adjacent cases with the same destination. We do this at all
13085 // optimization levels because it's cheap to do and will make codegen faster
13086 // if there are many clusters.
13087 sortAndRangeify(Clusters);
13088
13089 // The branch probablity of the peeled case.
13090 BranchProbability PeeledCaseProb = BranchProbability::getZero();
13091 MachineBasicBlock *PeeledSwitchMBB =
13092 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
13093
13094 // If there is only the default destination, jump there directly.
13095 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
13096 if (Clusters.empty()) {
13097 assert(PeeledSwitchMBB == SwitchMBB);
13098 SwitchMBB->addSuccessor(DefaultMBB);
13099 if (DefaultMBB != NextBlock(SwitchMBB)) {
13100 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
13101 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
13102 }
13103 return;
13104 }
13105
13106 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
13107 DAG.getBFI());
13108 SL->findBitTestClusters(Clusters, &SI);
13109
13110 LLVM_DEBUG({
13111 dbgs() << "Case clusters: ";
13112 for (const CaseCluster &C : Clusters) {
13113 if (C.Kind == CC_JumpTable)
13114 dbgs() << "JT:";
13115 if (C.Kind == CC_BitTests)
13116 dbgs() << "BT:";
13117
13118 C.Low->getValue().print(dbgs(), true);
13119 if (C.Low != C.High) {
13120 dbgs() << '-';
13121 C.High->getValue().print(dbgs(), true);
13122 }
13123 dbgs() << ' ';
13124 }
13125 dbgs() << '\n';
13126 });
13127
13128 assert(!Clusters.empty());
13129 SwitchWorkList WorkList;
13130 CaseClusterIt First = Clusters.begin();
13131 CaseClusterIt Last = Clusters.end() - 1;
13132 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
13133 // Scale the branchprobability for DefaultMBB if the peel occurs and
13134 // DefaultMBB is not replaced.
13135 if (PeeledCaseProb != BranchProbability::getZero() &&
13136 DefaultMBB == FuncInfo.getMBB(SI.getDefaultDest()))
13137 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
13138 WorkList.push_back(
13139 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
13140
13141 while (!WorkList.empty()) {
13142 SwitchWorkListItem W = WorkList.pop_back_val();
13143 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
13144
13145 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
13146 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
13147 // For optimized builds, lower large range as a balanced binary tree.
13148 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
13149 continue;
13150 }
13151
13152 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
13153 }
13154}
13155
13156void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
13157 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13158 auto DL = getCurSDLoc();
13159 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13160 setValue(&I, DAG.getStepVector(DL, ResultVT));
13161}
13162
13163void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
13164 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13165 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13166
13167 SDLoc DL = getCurSDLoc();
13168 SDValue V = getValue(I.getOperand(0));
13169 assert(VT == V.getValueType() && "Malformed vector.reverse!");
13170
13171 if (VT.isScalableVector()) {
13172 setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
13173 return;
13174 }
13175
13176 // Use VECTOR_SHUFFLE for the fixed-length vector
13177 // to maintain existing behavior.
13178 SmallVector<int, 8> Mask;
13179 unsigned NumElts = VT.getVectorMinNumElements();
13180 for (unsigned i = 0; i != NumElts; ++i)
13181 Mask.push_back(NumElts - 1 - i);
13182
13183 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
13184}
13185
13186void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
13187 unsigned Factor) {
13188 auto DL = getCurSDLoc();
13189 SDValue InVec = getValue(I.getOperand(0));
13190
13191 SmallVector<EVT, 4> ValueVTs;
13192 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
13193 ValueVTs);
13194
13195 EVT OutVT = ValueVTs[0];
13196 unsigned OutNumElts = OutVT.getVectorMinNumElements();
13197
13198 SmallVector<SDValue, 4> SubVecs(Factor);
13199 for (unsigned i = 0; i != Factor; ++i) {
13200 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
13201 SubVecs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
13202 DAG.getVectorIdxConstant(OutNumElts * i, DL));
13203 }
13204
13205 SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
13206 DAG.getVTList(ValueVTs), SubVecs);
13207 setValue(&I, Res);
13208}
13209
13210void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
13211 unsigned Factor) {
13212 auto DL = getCurSDLoc();
13213 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13214 EVT InVT = getValue(I.getOperand(0)).getValueType();
13215 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13216
13217 SmallVector<SDValue, 8> InVecs(Factor);
13218 for (unsigned i = 0; i < Factor; ++i) {
13219 InVecs[i] = getValue(I.getOperand(i));
13220 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
13221 "Expected VTs to be the same");
13222 }
13223
13224 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
13225 SDValue Res =
13226 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);
13227
13229 for (unsigned i = 0; i < Factor; ++i)
13230 Results[i] = Res.getValue(i);
13231
13232 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Results);
13233 setValue(&I, Res);
13234}
13235
13236void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
13237 SmallVector<EVT, 4> ValueVTs;
13238 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
13239 ValueVTs);
13240 unsigned NumValues = ValueVTs.size();
13241 if (NumValues == 0) return;
13242
13244 SDValue Op = getValue(I.getOperand(0));
13245
13246 for (unsigned i = 0; i != NumValues; ++i)
13247 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
13248 SDValue(Op.getNode(), Op.getResNo() + i));
13249
13251 DAG.getVTList(ValueVTs), Values));
13252}
13253
13254void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
13255 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13256 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13257
13258 SDLoc DL = getCurSDLoc();
13259 SDValue V1 = getValue(I.getOperand(0));
13260 SDValue V2 = getValue(I.getOperand(1));
13261 const bool IsLeft = I.getIntrinsicID() == Intrinsic::vector_splice_left;
13262
13263 // VECTOR_SHUFFLE doesn't support a scalable or non-constant mask.
13264 if (VT.isScalableVector() || !isa<ConstantInt>(I.getOperand(2))) {
13265 SDValue Offset = DAG.getZExtOrTrunc(
13266 getValue(I.getOperand(2)), DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
13267 setValue(&I, DAG.getNode(IsLeft ? ISD::VECTOR_SPLICE_LEFT
13269 DL, VT, V1, V2, Offset));
13270 return;
13271 }
13272 uint64_t Imm = cast<ConstantInt>(I.getOperand(2))->getZExtValue();
13273
13274 unsigned NumElts = VT.getVectorNumElements();
13275
13276 uint64_t Idx = IsLeft ? Imm : NumElts - Imm;
13277
13278 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
13279 SmallVector<int, 8> Mask;
13280 for (unsigned i = 0; i < NumElts; ++i)
13281 Mask.push_back(Idx + i);
13282 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
13283}
13284
13285// Consider the following MIR after SelectionDAG, which produces output in
13286// phyregs in the first case or virtregs in the second case.
13287//
13288// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
13289// %5:gr32 = COPY $ebx
13290// %6:gr32 = COPY $edx
13291// %1:gr32 = COPY %6:gr32
13292// %0:gr32 = COPY %5:gr32
13293//
13294// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
13295// %1:gr32 = COPY %6:gr32
13296// %0:gr32 = COPY %5:gr32
13297//
13298// Given %0, we'd like to return $ebx in the first case and %5 in the second.
13299// Given %1, we'd like to return $edx in the first case and %6 in the second.
13300//
13301// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
13302// to a single virtreg (such as %0). The remaining outputs monotonically
13303// increase in virtreg number from there. If a callbr has no outputs, then it
13304// should not have a corresponding callbr landingpad; in fact, the callbr
13305// landingpad would not even be able to refer to such a callbr.
13307 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
13308 // There is definitely at least one copy.
13309 assert(MI->getOpcode() == TargetOpcode::COPY &&
13310 "start of copy chain MUST be COPY");
13311 Reg = MI->getOperand(1).getReg();
13312
13313 // If the copied register in the first copy must be virtual.
13314 assert(Reg.isVirtual() && "expected COPY of virtual register");
13315 MI = MRI.def_begin(Reg)->getParent();
13316
13317 // There may be an optional second copy.
13318 if (MI->getOpcode() == TargetOpcode::COPY) {
13319 assert(Reg.isVirtual() && "expected COPY of virtual register");
13320 Reg = MI->getOperand(1).getReg();
13321 assert(Reg.isPhysical() && "expected COPY of physical register");
13322 } else {
13323 // The start of the chain must be an INLINEASM_BR.
13324 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13325 "end of copy chain MUST be INLINEASM_BR");
13326 }
13327
13328 return Reg;
13329}
13330
13331// We must do this walk rather than the simpler
13332// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
13333// otherwise we will end up with copies of virtregs only valid along direct
13334// edges.
13335void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
13336 SmallVector<EVT, 8> ResultVTs;
13337 SmallVector<SDValue, 8> ResultValues;
13338 const auto *CBR =
13339 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
13340
13341 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13342 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
13343 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
13344
13345 Register InitialDef = FuncInfo.ValueMap[CBR];
13346 SDValue Chain = DAG.getRoot();
13347
13348 // Re-parse the asm constraints string.
13349 TargetLowering::AsmOperandInfoVector TargetConstraints =
13350 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
13351 for (auto &T : TargetConstraints) {
13352 SDISelAsmOperandInfo OpInfo(T);
13353 if (OpInfo.Type != InlineAsm::isOutput)
13354 continue;
13355
13356 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
13357 // individual constraint.
13358 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
13359
13360 switch (OpInfo.ConstraintType) {
13363 // Fill in OpInfo.AssignedRegs.Regs.
13364 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
13365
13366 // getRegistersForValue may produce 1 to many registers based on whether
13367 // the OpInfo.ConstraintVT is legal on the target or not.
13368 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
13369 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
13370 if (OriginalDef.isPhysical())
13371 FuncInfo.MBB->addLiveIn(OriginalDef);
13372 // Update the assigned registers to use the original defs.
13373 Reg = OriginalDef;
13374 }
13375
13376 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
13377 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
13378 ResultValues.push_back(V);
13379 ResultVTs.push_back(OpInfo.ConstraintVT);
13380 break;
13381 }
13383 SDValue Flag;
13385 OpInfo, DAG);
13386 ++InitialDef;
13387 ResultValues.push_back(V);
13388 ResultVTs.push_back(OpInfo.ConstraintVT);
13389 break;
13390 }
13391 default:
13392 break;
13393 }
13394 }
13396 DAG.getVTList(ResultVTs), ResultValues);
13397 setValue(&I, V);
13398}
unsigned NumParts
return SDValue()
static unsigned getIntrinsicID(const SDNode *N)
bool IsScalable
const TargetLowering & TLI
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
dxil translate DXIL Translate Metadata
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition FastISel.cpp:947
#define Check(C,...)
static Value * getCondition(Instruction *I)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
static void computeConstraintToUse(const TargetLowering *TLI, TargetLowering::AsmOperandInfo &OpInfo)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
static bool isUndef(const MachineInstr &MI)
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file contains the declarations for metadata subclasses.
Type::TypeID TypeID
#define T
#define T1
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
#define P(N)
if(PassOpts->AAPipeline)
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
MachineInstr *& Lo
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static void failForInvalidBundles(const CallBase &I, StringRef Name, ArrayRef< uint32_t > AllowedBundles)
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, SelectionDAG &DAG)
Prepare DAG-level operands.
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static const GlobalValue * getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx, const MachineFunction &MF)
If V is the address of a describable global, possibly displaced by a constant, return the global and ...
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
DenseMap< const Argument *, std::pair< const AllocaInst *, const StoreInst * > > ArgCopyElisionMapTy
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< Register, TypeSize > > &Regs, const SDValue &N)
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static bool determineConstraints(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, const TargetMachine &TM, SelectionDAG &DAG, const BasicBlock *EHPadBB)
DetermineConstraints - Find the constraints to use for inline asm operands.
static bool constructOperandInfo(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, SelectionDAGBuilder &Builder, const TargetLowering &TLI, ExtraFlags &ExtraInfo)
Construct operand info objects.
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts, MVT RegisterVT, MVT PartVT)
Emit a fatal error if the broken down registers don't match part type or count.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static FPClassTest getNoFPClass(const Instruction &I)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const uint8_t *MatcherTable, size_t &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
TTI::PartialReductionExtendKind ExtendKind
uint16_t RegSizeInBits(const MCRegisterInfo &MRI, MCRegister RegNo)
Value * RHS
Value * LHS
The Input class is used to parse a yaml document into in-memory structs and vectors.
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static LLVM_ABI Semantics SemanticsToEnum(const llvm::fltSemantics &Sem)
Definition APFloat.cpp:185
static LLVM_ABI const fltSemantics * getArbitraryFPSemantics(StringRef Format)
Returns the fltSemantics for a given arbitrary FP format string, or nullptr if invalid.
Definition APFloat.cpp:6340
Class for arbitrary precision integers.
Definition APInt.h:78
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:330
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition Function.cpp:332
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition Argument.h:50
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
iterator end() const
Definition ArrayRef.h:130
size_t size() const
Get the array size.
Definition ArrayRef.h:141
iterator begin() const
Definition ArrayRef.h:129
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
@ Add
*p = old + v
@ FAdd
*p = old + v
@ USubCond
Subtract only if no unsigned overflow.
@ FMinimum
*p = minimum(old, v) minimum matches the behavior of llvm.minimum.
@ Min
*p = old <signed v ? old : v
@ Sub
*p = old - v
@ And
*p = old & v
@ Xor
*p = old ^ v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ FMaximum
*p = maximum(old, v) maximum matches the behavior of llvm.maximum.
@ FSub
*p = old - v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMaximumNum
*p = maximumnum(old, v) maximumnum matches the behavior of llvm.maximumnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
@ FMinimumNum
*p = minimumnum(old, v) minimumnum matches the behavior of llvm.minimumnum.
@ Nand
*p = ~(old & v)
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:448
LLVM Basic Block Representation.
Definition BasicBlock.h:62
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
InstListType::const_iterator const_iterator
Definition BasicBlock.h:171
LLVM_ABI bool isEntryBlock() const
Return true if this is the entry block of the containing function.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
This class represents a no-op cast from one type to another.
The address of a basic block.
Definition Constants.h:1088
Analysis providing branch probability information.
LLVM_ABI BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
LLVM_ABI bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static constexpr BranchProbability getOne()
static uint32_t getDenominator()
static constexpr BranchProbability getUnknown()
static constexpr BranchProbability getZero()
uint32_t getNumerator() const
LLVM_ABI uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
LLVM_ABI bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition InstrTypes.h:728
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
Conditional Branch instruction.
Class for constant bytes.
Definition Constants.h:281
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition Constants.h:755
A constant value that is initialized with an expression using other constant values.
Definition Constants.h:1316
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
A signed pointer, in the ptrauth sense.
Definition Constants.h:1223
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition Constants.h:674
This is an important base class in LLVM.
Definition Constant.h:44
This is the common base class for constrained floating point intrinsics.
LLVM_ABI std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
LLVM_ABI unsigned getNonMetadataArgCount() const
DWARF expression.
LLVM_ABI bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static LLVM_ABI std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
LLVM_ABI uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static LLVM_ABI const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static LLVM_ABI DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
Base class for variables.
LLVM_ABI std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool isBigEndian() const
Definition DataLayout.h:218
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LLVM_ABI Value * getVariableLocationOp(unsigned OpIdx) const
DIExpression * getExpression() const
DILocalVariable * getVariable() const
LLVM_ABI iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
A debug info location.
Definition DebugLoc.h:126
LLVM_ABI DILocation * getInlinedAt() const
Definition DebugLoc.cpp:58
bool empty() const
Definition DenseMap.h:716
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT, true > const_iterator
Definition DenseMap.h:679
void reserve(size_type NumEntries)
Grow the DenseMap so that it can contain at least NumEntries items before resizing again.
Definition DenseMap.h:721
iterator end()
Definition DenseMap.h:686
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
Diagnostic information for inline asm reporting.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:312
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
Lightweight error class with error context and mandatory checking.
Definition Error.h:159
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition FastISel.h:67
bool allowReassoc() const
Flag queries.
Definition FMF.h:64
An instruction for ordering other memory operations.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
MachineBasicBlock * getMBB(const BasicBlock *BB) const
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
MachineBasicBlock * MBB
MBB - The current block.
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition Function.h:803
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:705
bool hasParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const
check if an attributes is in the list of attributes.
Definition Function.cpp:743
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:329
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition Function.h:252
size_t arg_size() const
Definition Function.h:895
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
Garbage collection metadata for a single function.
Definition GCMetadata.h:80
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
bool isInBounds() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
void setMemConstraint(ConstraintCode C)
setMemConstraint - Augment an existing flag with the constraint code for a memory constraint.
Definition InlineAsm.h:417
This instruction inserts a struct field of array element value into an aggregate value.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
A helper class to return the specified delimiter string after the first invocation of operator String...
An instruction for reading from memory.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
static LocationSize upperBound(uint64_t Value)
bool usesPerInvokeEHLabels(ExceptionHandling EH) const
Returns true if the exception tables reference the per-invoke EH labels emitted around invokes.
Definition MCAsmInfo.h:692
LLVM_ABI MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
LLVM_ABI MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
const MCAsmInfo & getAsmInfo() const
Definition MCContext.h:411
unsigned getID() const
getID() - Return the register class ID number.
bool isAllocatable() const
isAllocatable - Return true if this register class may be used to create virtual registers.
const MCPhysReg * iterator
iterator begin() const
begin/end - Return all of the registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
iterator end() const
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition MCSymbol.h:42
Metadata node.
Definition Metadata.h:1081
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:613
Machine Value Type.
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
std::string getString() const
Return the value type as a string, e.g. i32.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHContTarget(bool V=true)
Indicates if this is a target of Windows EH Continuation Guard.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
MachineInstrBundleIterator< MachineInstr > iterator
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
void setHasEHContTarget(bool V)
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI MCRegister getLiveInPhysReg(Register VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition MapVector.h:119
bool contains(const KeyT &Key) const
Definition MapVector.h:149
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Definition Module.cpp:377
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:887
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(Register VReg)
static SDDbgOperand fromConst(const Value *Const)
static SDDbgOperand fromGlobalAddr(const GlobalValue *GV)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const CondBrInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
DenseMap< const Constant *, Register > ConstantsOut
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
SDValue lowerStartEH(SDValue Chain, const BasicBlock *EHPadBB, MCSymbol *&BeginLabel)
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
const TargetTransformInfo * TTI
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
SDValue lowerNoFPClassToAssertNoFPClass(SelectionDAG &DAG, const Instruction &I, SDValue Op)
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void setValueToPoison(const Value *V, const SDLoc &dl)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, Register Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool canTailCall(const CallBase &CB) const
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void CopyValueToVirtualRegister(const Value *V, Register Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, const TargetLibraryInfo *li, const TargetTransformInfo &TTI)
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
SDValue getFPOperationRoot(fp::ExceptionBehavior EB)
Return the current virtual root of the Selection DAG, flushing PendingConstrainedFP or PendingConstra...
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineRegisterInfo * RegInfo
std::unique_ptr< SwiftErrorValueTracking > SwiftError
virtual void emitFunctionEntryCode()
std::unique_ptr< SelectionDAGBuilder > SDB
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemccpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue C, SDValue Size, const CallInst *CI) const
Emit target-specific code that performs a memccpy, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, const CallInst *CI) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrstr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, const CallInst *CI) const
Emit target-specific code that performs a strstr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo, const CallInst *CI) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, const CallInst *CI) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy, const CallInst *CI) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI bool shouldOptForSize() const
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
LLVM_ABI void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
LLVM_ABI SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void resize(size_type N)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitFunctionBasedCheckStackProtector() const
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
Multiway switch.
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Provides information about what library functions are available for the current target.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Returns the desired alignment for ByVal or InAlloca aggregate function arguments in the caller parame...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
virtual bool isAtomicAlignmentSupported(Align Alignment, uint64_t SizeInBytes) const
Return true if the target supports an atomic access of SizeInBytes bytes at the given Alignment.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
virtual Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual bool useStackGuardMixFP() const
If this function returns true, stack protection checks should mix the frame pointer (or whichever poi...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr, CodeGenOptLevel OptLevel=CodeGenOptLevel::Default) const
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
MachineMemOperand::Flags getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
virtual Value * getSDagStackGuard(const Module &M, const LibcallLoweringInfo &Libcalls) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
virtual Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_UDIV, ISD::VP_SDIV,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual SDValue emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
Definition Type.cpp:170
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:272
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:296
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
bool isTokenTy() const
Return true if this is 'token'.
Definition Type.h:231
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:303
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
Unconditional Branch instruction.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_iterator op_begin()
Definition User.h:259
unsigned getNumOperands() const
Definition User.h:229
op_iterator op_end()
Definition User.h:261
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
This is the common base class for vector predication intrinsics.
static LLVM_ABI std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
LLVM_ABI MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:707
bool use_empty() const
Definition Value.h:348
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
const ParentTy * getParent() const
Definition ilist_node.h:34
A raw_ostream that writes to an std::string.
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ CONVERGENCECTRL_ANCHOR
The llvm.experimental.convergence.* intrinsics.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition ISDOpcodes.h:47
@ SET_FPENV
Sets the current floating-point environment.
@ ATOMIC_LOAD_FMINIMUMNUM
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ COND_LOOP
COND_LOOP is a conditional branch to self, used for implementing efficient conditional traps.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition ISDOpcodes.h:170
@ VECREDUCE_FMINIMUMNUM
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ STACKADDRESS
STACKADDRESS - Represents the llvm.stackaddress intrinsic.
Definition ISDOpcodes.h:129
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:397
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:403
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ ATOMIC_LOAD_USUB_COND
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition ISDOpcodes.h:174
@ GlobalAddress
Definition ISDOpcodes.h:90
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
Definition ISDOpcodes.h:665
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
Definition ISDOpcodes.h:533
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
Definition ISDOpcodes.h:520
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:410
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ ATOMIC_LOAD_USUB_SAT
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition ISDOpcodes.h:158
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ SET_ROUNDING
Set rounding mode.
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition ISDOpcodes.h:119
@ CONVERGENCECTRL_ENTRY
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PARTIAL_REDUCE_FMLA
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:640
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
@ ATOMIC_LOAD_FMAXIMUM
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
Definition ISDOpcodes.h:80
@ PtrAuthGlobalAddress
A ptrauth constant.
Definition ISDOpcodes.h:102
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition ISDOpcodes.h:50
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition ISDOpcodes.h:141
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition ISDOpcodes.h:137
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition ISDOpcodes.h:389
@ PATCHPOINT
The llvm.experimental.patchpoint.
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ ATOMIC_LOAD_FMINIMUM
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
Definition ISDOpcodes.h:669
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition ISDOpcodes.h:650
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:416
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ ATOMIC_LOAD_FMAXIMUMNUM
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition ISDOpcodes.h:152
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:112
@ ATOMIC_LOAD_UDEC_WRAP
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
Definition ISDOpcodes.h:807
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:505
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ RELOC_NONE
Issue a no-op relocation against a given symbol at the current location.
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition ISDOpcodes.h:759
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
Definition ISDOpcodes.h:673
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition ISDOpcodes.h:430
@ STACKMAP
The llvm.experimental.stackmap intrinsic.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition ISDOpcodes.h:715
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition ISDOpcodes.h:124
@ CLEAR_CACHE
llvm.clear_cache intrinsic Operands: Input Chain, Start Addres, End Address Outputs: Output Chain
@ CONVERGENCECTRL_LOOP
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ VECREDUCE_FMINIMUM
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition ISDOpcodes.h:164
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ BRCOND
BRCOND - Conditional branch.
@ VECREDUCE_SEQ_FMUL
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ ATOMIC_LOAD_UINC_WRAP
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
Definition ISDOpcodes.h:645
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:629
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ CTTZ_ELTS_ZERO_POISON
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
Definition ISDOpcodes.h:775
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Flag
These should be considered private to the implementation of the MCInstrDesc class.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_VScale()
Matches a call to llvm.vscale().
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
Offsets
Offsets in bytes from the start of the input buffer.
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
LLVM_ABI void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
std::vector< CaseCluster > CaseClusterVector
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
CaseClusterVector::iterator CaseClusterIt
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition Dwarf.h:149
ExceptionBehavior
Exception behavior used for floating point operations.
Definition FPEnv.h:39
@ ebStrict
This corresponds to "fpexcept.strict".
Definition FPEnv.h:42
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition FPEnv.h:41
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition FPEnv.h:40
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)
Extract a Value from Metadata, allowing null.
Definition Metadata.h:694
constexpr float log2ef
Definition MathExtras.h:52
constexpr double e
constexpr float ln2f
Definition MathExtras.h:50
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:339
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
LLVM_ABI bool canDescribeGlobalAddressInLocationList(const MachineFunction &MF)
Test if the debug info for MF can name a describable global address in a location list too,...
Definition Analysis.cpp:642
LLVM_ABI ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition Analysis.cpp:248
LLVM_ABI cl::opt< bool > ProfcheckDisableMetadataFixes
Definition LoopInfo.cpp:61
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
SDValue peekThroughFreeze(SDValue V)
Return the non-frozen source operand of V if it exists.
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
RelativeUniformCounterPtr Values
Definition InstrProf.h:91
@ Unknown
Not known to have no common set bits.
LLVM_ABI void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs=nullptr, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition Analysis.cpp:121
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Done
Definition Threading.h:60
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
LLVM_ABI void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
LLVM_ABI bool isExceptionPointerAndSelectorType(Type *Ty)
Return true if landingpad result type Ty is a struct of an exception pointer (pointer or integer) and...
Definition Analysis.cpp:201
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
static ConstantRange getRange(Value *Op, SCCPSolver &Solver, const SmallPtrSetImpl< Value * > &InsertedValues)
Helper for getting ranges from Solver.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
auto cast_or_null(const Y &Val)
Definition Casting.h:714
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
gep_type_iterator gep_type_end(const User *GEP)
LLVM_ABI LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
Definition STLExtras.h:2189
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
Definition STLExtras.h:1167
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
LLVM_ABI void ComputeValueTypes(const DataLayout &DL, Type *Ty, SmallVectorImpl< Type * > &Types, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
Given an LLVM IR type, compute non-aggregate subtypes.
Definition Analysis.cpp:74
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition STLExtras.h:869
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
generic_gep_type_iterator<> gep_type_iterator
auto succ_size(const MachineBasicBlock *BB)
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition STLExtras.h:300
LLVM_ABI ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition Analysis.cpp:214
LLVM_ABI MDNode * getMemCacheHint(const Instruction &I, unsigned OperandNo=0)
Return the hint payload for operand OperandNo of I, or nullptr if absent.
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition Local.cpp:2353
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
LLVM_ABI bool isOnlyUsedInZeroEqualityComparison(const Instruction *CtxI)
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
LLVM_ABI bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition Analysis.cpp:656
DWARFExpression::Operation Op
@ Dynamic
Denotes mode unknown at compile time.
LLVM_ABI ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition Analysis.cpp:236
LLVM_ABI const GlobalValue * getDescribableGlobalAddress(const Constant *C, int64_t &Offset, const MachineFunction &MF)
If C is the address of a global, possibly displaced by a constant, return that global and set Offset ...
Definition Analysis.cpp:623
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
LLVM_ABI std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition FPEnv.cpp:25
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
LLVM_ABI GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition Analysis.cpp:183
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
LLVM_ABI Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
LLVM_ABI unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition Analysis.cpp:35
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:347
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
#define NC
Definition regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
uint64_t getScalarStoreSize() const
Definition ValueTypes.h:440
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:98
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
Definition ValueTypes.h:212
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isFixedLengthVector() const
Definition ValueTypes.h:214
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:330
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
Definition ValueTypes.h:136
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition InlineAsm.h:128
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:262
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
A lightweight accessor for an operand bundle meant to be passed around by value.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< std::pair< Register, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< Register, 4 > Regs
This list holds the registers assigned to the values.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
void setUnpredictable(bool b)
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition MapVector.h:344
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
Register Reg
The virtual register containing the index of the jump table entry to jump to.
MachineBasicBlock * Default
The MBB of the default bb, which is a successor of the range check MBB.
unsigned JTI
The JumpTableIndex for this jump table in the function.
MachineBasicBlock * MBB
The MBB into which to emit the code for the indirect jump.
std::optional< SDLoc > SL
The debug location of the instruction this JumpTable was produced from.
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setDeactivationSymbol(GlobalValue *Sym)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
Type * OrigRetTy
Original unlegalized return type.
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
LLVM_ABI void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)