LLVM 24.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsBPF.h"
82#include "llvm/IR/IntrinsicsWebAssembly.h"
83#include "llvm/IR/LLVMContext.h"
86#include "llvm/IR/Metadata.h"
87#include "llvm/IR/Module.h"
88#include "llvm/IR/Operator.h"
91#include "llvm/IR/Statepoint.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCAsmInfo.h"
96#include "llvm/MC/MCContext.h"
101#include "llvm/Support/Debug.h"
109#include <cstddef>
110#include <limits>
111#include <optional>
112#include <tuple>
113
114using namespace llvm;
115using namespace PatternMatch;
116using namespace SwitchCG;
117
118#define DEBUG_TYPE "isel"
119
120/// LimitFloatPrecision - Generate low-precision inline sequences for
121/// some float libcalls (6, 8 or 12 bits).
122static unsigned LimitFloatPrecision;
123
124static cl::opt<bool>
125 InsertAssertAlign("insert-assert-align", cl::init(true),
126 cl::desc("Insert the experimental `assertalign` node."),
128
130 LimitFPPrecision("limit-float-precision",
131 cl::desc("Generate low-precision inline sequences "
132 "for some float libcalls"),
134 cl::init(0));
135
137 "switch-peel-threshold", cl::Hidden, cl::init(66),
138 cl::desc("Set the case probability threshold for peeling the case from a "
139 "switch statement. A value greater than 100 will void this "
140 "optimization"));
141
142// Limit the width of DAG chains. This is important in general to prevent
143// DAG-based analysis from blowing up. For example, alias analysis and
144// load clustering may not complete in reasonable time. It is difficult to
145// recognize and avoid this situation within each individual analysis, and
146// future analyses are likely to have the same behavior. Limiting DAG width is
147// the safe approach and will be especially important with global DAGs.
148//
149// MaxParallelChains default is arbitrarily high to avoid affecting
150// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
151// sequence over this should have been converted to llvm.memcpy by the
152// frontend. It is easy to induce this behavior with .ll code such as:
153// %buffer = alloca [4096 x i8]
154// %data = load [4096 x i8]* %argPtr
155// store [4096 x i8] %data, [4096 x i8]* %buffer
156static const unsigned MaxParallelChains = 64;
157
159 const SDValue *Parts, unsigned NumParts,
160 MVT PartVT, EVT ValueVT, const Value *V,
161 SDValue InChain,
162 std::optional<CallingConv::ID> CC);
163
164/// getCopyFromParts - Create a value that contains the specified legal parts
165/// combined into the value they represent. If the parts combine to a type
166/// larger than ValueVT then AssertOp can be used to specify whether the extra
167/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
168/// (ISD::AssertSext).
169static SDValue
170getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
171 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
172 SDValue InChain,
173 std::optional<CallingConv::ID> CC = std::nullopt,
174 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
175 // Let the target assemble the parts if it wants to
177 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
178 PartVT, ValueVT, CC))
179 return Val;
180
181 if (ValueVT.isVector())
182 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
183 InChain, CC);
184
185 assert(NumParts > 0 && "No parts to assemble!");
186 SDValue Val = Parts[0];
187
188 if (NumParts > 1) {
189 // Assemble the value from multiple parts.
190 if (ValueVT.isInteger()) {
191 unsigned PartBits = PartVT.getSizeInBits();
192 unsigned ValueBits = ValueVT.getSizeInBits();
193
194 // Assemble the power of 2 part.
195 unsigned RoundParts = llvm::bit_floor(NumParts);
196 unsigned RoundBits = PartBits * RoundParts;
197 EVT RoundVT = RoundBits == ValueBits ?
198 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
199 SDValue Lo, Hi;
200
201 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
202
203 if (RoundParts > 2) {
204 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
205 InChain);
206 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
207 PartVT, HalfVT, V, InChain);
208 } else {
209 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
210 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
211 }
212
213 if (DAG.getDataLayout().isBigEndian())
214 std::swap(Lo, Hi);
215
216 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
217
218 if (RoundParts < NumParts) {
219 // Assemble the trailing non-power-of-2 part.
220 unsigned OddParts = NumParts - RoundParts;
221 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
222 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
223 OddVT, V, InChain, CC);
224
225 // Combine the round and odd parts.
226 Lo = Val;
227 if (DAG.getDataLayout().isBigEndian())
228 std::swap(Lo, Hi);
229 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
230 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
231 Hi = DAG.getNode(
232 ISD::SHL, DL, TotalVT, Hi,
233 DAG.getShiftAmountConstant(Lo.getValueSizeInBits(), TotalVT, DL));
234 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
235 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
236 }
237 } else if (PartVT.isFloatingPoint()) {
238 // FP split into multiple FP parts (for ppcf128)
239 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
240 "Unexpected split");
241 SDValue Lo, Hi;
242 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
243 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
244 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
245 std::swap(Lo, Hi);
246 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
247 } else {
248 // FP split into integer parts (soft fp)
249 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
250 !PartVT.isVector() && "Unexpected split");
251 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
252 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
253 InChain, CC);
254 }
255 }
256
257 // There is now one part, held in Val. Correct it to match ValueVT.
258 // PartEVT is the type of the register class that holds the value.
259 // ValueVT is the type of the inline asm operation.
260 EVT PartEVT = Val.getValueType();
261
262 if (PartEVT == ValueVT)
263 return Val;
264
265 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
266 ValueVT.bitsLT(PartEVT)) {
267 // For an FP value in an integer part, we need to truncate to the right
268 // width first.
269 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
270 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
271 }
272
273 // Handle types that have the same size.
274 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
275 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
276
277 // Handle types with different sizes.
278 if (PartEVT.isInteger() && ValueVT.isInteger()) {
279 if (ValueVT.bitsLT(PartEVT)) {
280 // For a truncate, see if we have any information to
281 // indicate whether the truncated bits will always be
282 // zero or sign-extension.
283 if (AssertOp)
284 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
285 DAG.getValueType(ValueVT));
286 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
287 }
288 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
289 }
290
291 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
292 // FP_ROUND's are always exact here.
293 if (ValueVT.bitsLT(Val.getValueType())) {
294
295 SDValue NoChange =
296 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
297
298 if (DAG.getMachineFunction().getFunction().getAttributes().hasFnAttr(
299 llvm::Attribute::StrictFP)) {
300 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
301 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
302 NoChange);
303 }
304
305 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
306 }
307
308 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
309 }
310
311 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
312 // then truncating.
313 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
314 ValueVT.bitsLT(PartEVT)) {
315 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
316 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
317 }
318
319 report_fatal_error("Unknown mismatch in getCopyFromParts!");
320}
321
323 const Twine &ErrMsg) {
325 if (!I)
326 return Ctx.emitError(ErrMsg);
327
328 if (const CallInst *CI = dyn_cast<CallInst>(I))
329 if (CI->isInlineAsm()) {
330 return Ctx.diagnose(DiagnosticInfoInlineAsm(
331 *CI, ErrMsg + ", possible invalid constraint for vector type"));
332 }
333
334 return Ctx.emitError(I, ErrMsg);
335}
336
337/// Emit a fatal error if the broken down registers don't match part type
338/// or count.
339static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts,
340 MVT RegisterVT, MVT PartVT) {
341 if (NumRegs == NumParts && RegisterVT == PartVT)
342 return;
343
345 Twine("Part count doesn't match vector breakdown! ") + Twine(NumRegs) +
346 " registers, " + Twine(NumParts) + " parts, " + RegisterVT.getString() +
347 " RegisterVT, " + PartVT.getString() + " PartVT");
348}
349
350/// getCopyFromPartsVector - Create a value that contains the specified legal
351/// parts combined into the value they represent. If the parts combine to a
352/// type larger than ValueVT then AssertOp can be used to specify whether the
353/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
354/// ValueVT (ISD::AssertSext).
356 const SDValue *Parts, unsigned NumParts,
357 MVT PartVT, EVT ValueVT, const Value *V,
358 SDValue InChain,
359 std::optional<CallingConv::ID> CallConv) {
360 assert(ValueVT.isVector() && "Not a vector value");
361 assert(NumParts > 0 && "No parts to assemble!");
362 const bool IsABIRegCopy = CallConv.has_value();
363
365 SDValue Val = Parts[0];
366
367 // Handle a multi-element vector.
368 if (NumParts > 1) {
369 EVT IntermediateVT;
370 MVT RegisterVT;
371 unsigned NumIntermediates;
372 unsigned NumRegs;
373
374 if (IsABIRegCopy) {
375 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
376 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
377 NumIntermediates, RegisterVT);
378 } else {
379 NumRegs =
380 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
381 NumIntermediates, RegisterVT);
382 }
383
384 ensureMatchedVecRegParts(NumRegs, NumParts, RegisterVT, PartVT);
385 assert(RegisterVT.getSizeInBits() ==
386 Parts[0].getSimpleValueType().getSizeInBits() &&
387 "Part type sizes don't match!");
388
389 // Assemble the parts into intermediate operands.
390 SmallVector<SDValue, 8> Ops(NumIntermediates);
391 if (NumIntermediates == NumParts) {
392 // If the register was not expanded, truncate or copy the value,
393 // as appropriate.
394 for (unsigned i = 0; i != NumParts; ++i)
395 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
396 V, InChain, CallConv);
397 } else if (NumParts > 0) {
398 // If the intermediate type was expanded, build the intermediate
399 // operands from the parts.
400 assert(NumParts % NumIntermediates == 0 &&
401 "Must expand into a divisible number of parts!");
402 unsigned Factor = NumParts / NumIntermediates;
403 for (unsigned i = 0; i != NumIntermediates; ++i)
404 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
405 IntermediateVT, V, InChain, CallConv);
406 }
407
408 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
409 // intermediate operands.
410 EVT BuiltVectorTy =
411 IntermediateVT.isVector()
413 *DAG.getContext(), IntermediateVT.getScalarType(),
414 IntermediateVT.getVectorElementCount() * NumParts)
416 IntermediateVT.getScalarType(),
417 NumIntermediates);
418 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
420 DL, BuiltVectorTy, Ops);
421 }
422
423 // There is now one part, held in Val. Correct it to match ValueVT.
424 EVT PartEVT = Val.getValueType();
425
426 if (PartEVT == ValueVT)
427 return Val;
428
429 if (PartEVT.isVector()) {
430 // Vector/Vector bitcast.
431 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
432 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
433
434 // If the parts vector has more elements than the value vector, then we
435 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
436 // Extract the elements we want.
437 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
440 (PartEVT.getVectorElementCount().isScalable() ==
441 ValueVT.getVectorElementCount().isScalable()) &&
442 "Cannot narrow, it would be a lossy transformation");
443 PartEVT =
445 ValueVT.getVectorElementCount());
446 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
447 DAG.getVectorIdxConstant(0, DL));
448 if (PartEVT == ValueVT)
449 return Val;
450 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
451 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
452
453 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
454 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
455 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
456 }
457
458 // Promoted vector extract
459 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
460 }
461
462 // Trivial bitcast if the types are the same size and the destination
463 // vector type is legal.
464 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
465 TLI.isTypeLegal(ValueVT))
466 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
467
468 if (ValueVT.getVectorNumElements() != 1) {
469 // Certain ABIs require that vectors are passed as integers. For vectors
470 // are the same size, this is an obvious bitcast.
471 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
472 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
473 } else if (ValueVT.bitsLT(PartEVT)) {
474 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
475 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
476 // Drop the extra bits.
477 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
478 return DAG.getBitcast(ValueVT, Val);
479 }
480
482 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
483 return DAG.getUNDEF(ValueVT);
484 }
485
486 // Handle cases such as i8 -> <1 x i1>
487 EVT ValueSVT = ValueVT.getVectorElementType();
488 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
489 unsigned ValueSize = ValueSVT.getSizeInBits();
490 if (ValueSize == PartEVT.getSizeInBits()) {
491 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
492 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
493 // It's possible a scalar floating point type gets softened to integer and
494 // then promoted to a larger integer. If PartEVT is the larger integer
495 // we need to truncate it and then bitcast to the FP type.
496 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
497 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
498 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
499 Val = DAG.getBitcast(ValueSVT, Val);
500 } else {
501 Val = ValueVT.isFloatingPoint()
502 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
503 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
504 }
505 }
506
507 return DAG.getBuildVector(ValueVT, DL, Val);
508}
509
510static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
511 SDValue Val, SDValue *Parts, unsigned NumParts,
512 MVT PartVT, const Value *V,
513 std::optional<CallingConv::ID> CallConv);
514
515/// getCopyToParts - Create a series of nodes that contain the specified value
516/// split into legal parts. If the parts contain more bits than Val, then, for
517/// integers, ExtendKind can be used to specify how to generate the extra bits.
518static void
520 unsigned NumParts, MVT PartVT, const Value *V,
521 std::optional<CallingConv::ID> CallConv = std::nullopt,
523 // Let the target split the parts if it wants to
525 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
526 CallConv))
527 return;
528 EVT ValueVT = Val.getValueType();
529
530 // Handle the vector case separately.
531 if (ValueVT.isVector())
532 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
533 CallConv);
534
535 unsigned OrigNumParts = NumParts;
537 "Copying to an illegal type!");
538
539 if (NumParts == 0)
540 return;
541
542 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
543 EVT PartEVT = PartVT;
544 if (PartEVT == ValueVT) {
545 assert(NumParts == 1 && "No-op copy with multiple parts!");
546 Parts[0] = Val;
547 return;
548 }
549
550 unsigned PartBits = PartVT.getSizeInBits();
551 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
552 // If the parts cover more bits than the value has, promote the value.
553 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
554 assert(NumParts == 1 && "Do not know what to promote to!");
555 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
556 } else {
557 if (ValueVT.isFloatingPoint()) {
558 // FP values need to be bitcast, then extended if they are being put
559 // into a larger container.
560 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
561 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
562 }
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
567 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
570 }
571 } else if (PartBits == ValueVT.getSizeInBits()) {
572 // Different types of the same size.
573 assert(NumParts == 1 && PartEVT != ValueVT);
574 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
575 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
576 // If the parts cover less bits than value has, truncate the value.
577 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
578 ValueVT.isInteger() &&
579 "Unknown mismatch!");
580 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
581 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
582 if (PartVT == MVT::x86mmx)
583 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
584 }
585
586 // The value may have changed - recompute ValueVT.
587 ValueVT = Val.getValueType();
588 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
589 "Failed to tile the value with PartVT!");
590
591 if (NumParts == 1) {
592 if (PartEVT != ValueVT) {
594 "scalar-to-vector conversion failed");
595 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
596 }
597
598 Parts[0] = Val;
599 return;
600 }
601
602 // Expand the value into multiple parts.
603 if (NumParts & (NumParts - 1)) {
604 // The number of parts is not a power of 2. Split off and copy the tail.
605 assert(PartVT.isInteger() && ValueVT.isInteger() &&
606 "Do not know what to expand to!");
607 unsigned RoundParts = llvm::bit_floor(NumParts);
608 unsigned RoundBits = RoundParts * PartBits;
609 unsigned OddParts = NumParts - RoundParts;
610 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
611 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
612
613 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
614 CallConv);
615
616 if (DAG.getDataLayout().isBigEndian())
617 // The odd parts were reversed by getCopyToParts - unreverse them.
618 std::reverse(Parts + RoundParts, Parts + NumParts);
619
620 NumParts = RoundParts;
621 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
622 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
623 }
624
625 // The number of parts is a power of 2. Repeatedly bisect the value using
626 // EXTRACT_ELEMENT.
627 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
629 ValueVT.getSizeInBits()),
630 Val);
631
632 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
633 for (unsigned i = 0; i < NumParts; i += StepSize) {
634 unsigned ThisBits = StepSize * PartBits / 2;
635 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
636 SDValue &Part0 = Parts[i];
637 SDValue &Part1 = Parts[i+StepSize/2];
638
639 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
640 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
641 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
642 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
643
644 if (ThisBits == PartBits && ThisVT != PartVT) {
645 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
646 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
647 }
648 }
649 }
650
651 if (DAG.getDataLayout().isBigEndian())
652 std::reverse(Parts, Parts + OrigNumParts);
653}
654
656 const SDLoc &DL, EVT PartVT) {
657 if (!PartVT.isVector())
658 return SDValue();
659
660 EVT ValueVT = Val.getValueType();
661 EVT PartEVT = PartVT.getVectorElementType();
662 EVT ValueEVT = ValueVT.getVectorElementType();
663 ElementCount PartNumElts = PartVT.getVectorElementCount();
664 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
665
666 // We only support widening vectors with equivalent element types and
667 // fixed/scalable properties. If a target needs to widen a fixed-length type
668 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
669 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
670 PartNumElts.isScalable() != ValueNumElts.isScalable())
671 return SDValue();
672
673 // Have a try for bf16 because some targets share its ABI with fp16.
674 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
676 "Cannot widen to illegal type");
677 Val = DAG.getNode(
679 ValueVT.changeVectorElementType(*DAG.getContext(), MVT::f16), Val);
680 } else if (PartEVT != ValueEVT) {
681 return SDValue();
682 }
683
684 // Widening a scalable vector to another scalable vector is done by inserting
685 // the vector into a larger undef one.
686 if (PartNumElts.isScalable())
687 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
688 Val, DAG.getVectorIdxConstant(0, DL));
689
690 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
691 // undef elements.
693 DAG.ExtractVectorElements(Val, Ops);
694 SDValue EltUndef = DAG.getUNDEF(PartEVT);
695 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
696
697 // FIXME: Use CONCAT for 2x -> 4x.
698 return DAG.getBuildVector(PartVT, DL, Ops);
699}
700
701/// getCopyToPartsVector - Create a series of nodes that contain the specified
702/// value split into legal parts.
703static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
704 SDValue Val, SDValue *Parts, unsigned NumParts,
705 MVT PartVT, const Value *V,
706 std::optional<CallingConv::ID> CallConv) {
707 EVT ValueVT = Val.getValueType();
708 assert(ValueVT.isVector() && "Not a vector");
710 const bool IsABIRegCopy = CallConv.has_value();
711
712 if (NumParts == 1) {
713 EVT PartEVT = PartVT;
714 if (PartEVT == ValueVT) {
715 // Nothing to do.
716 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
717 // Bitconvert vector->vector case.
718 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
719 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
720 Val = Widened;
721 } else if (PartVT.isVector() &&
723 ValueVT.getVectorElementType()) &&
724 PartEVT.getVectorElementCount() ==
725 ValueVT.getVectorElementCount()) {
726
727 // Promoted vector extract
728 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
729 } else if (PartEVT.isVector() &&
730 PartEVT.getVectorElementType() !=
731 ValueVT.getVectorElementType() &&
732 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
734 // Combination of widening and promotion.
735 EVT WidenVT =
737 PartVT.getVectorElementCount());
738 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
739 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
740 } else {
741 // Don't extract an integer from a float vector. This can happen if the
742 // FP type gets softened to integer and then promoted. The promotion
743 // prevents it from being picked up by the earlier bitcast case.
744 if (ValueVT.getVectorElementCount().isScalar() &&
745 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
746 // If we reach this condition and PartVT is FP, this means that
747 // ValueVT is also FP and both have a different size, otherwise we
748 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
749 // would be invalid since that would mean the smaller FP type has to
750 // be extended to the larger one.
751 if (PartVT.isFloatingPoint()) {
752 Val = DAG.getBitcast(ValueVT.getScalarType(), Val);
753 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
754 } else
755 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
756 DAG.getVectorIdxConstant(0, DL));
757 } else {
758 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
759 assert(PartVT.getFixedSizeInBits() > ValueSize &&
760 "lossy conversion of vector to scalar type");
761 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
762 Val = DAG.getBitcast(IntermediateType, Val);
763 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
764 }
765 }
766
767 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
768 Parts[0] = Val;
769 return;
770 }
771
772 // Handle a multi-element vector.
773 EVT IntermediateVT;
774 MVT RegisterVT;
775 unsigned NumIntermediates;
776 unsigned NumRegs;
777 if (IsABIRegCopy) {
778 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
779 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
780 RegisterVT);
781 } else {
782 NumRegs =
783 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
784 NumIntermediates, RegisterVT);
785 }
786
787 ensureMatchedVecRegParts(NumRegs, NumParts, RegisterVT, PartVT);
788 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
789 "Mixing scalable and fixed vectors when copying in parts");
790
791 std::optional<ElementCount> DestEltCnt;
792
793 if (IntermediateVT.isVector())
794 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
795 else
796 DestEltCnt = ElementCount::getFixed(NumIntermediates);
797
798 EVT BuiltVectorTy = EVT::getVectorVT(
799 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
800
801 if (ValueVT == BuiltVectorTy) {
802 // Nothing to do.
803 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
804 // Bitconvert vector->vector case.
805 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
806 } else {
807 if (BuiltVectorTy.getVectorElementType().bitsGT(
808 ValueVT.getVectorElementType())) {
809 // Integer promotion.
810 ValueVT = EVT::getVectorVT(*DAG.getContext(),
811 BuiltVectorTy.getVectorElementType(),
812 ValueVT.getVectorElementCount());
813 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
814 }
815
816 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
817 Val = Widened;
818 }
819 }
820
821 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
822
823 // Split the vector into intermediate operands.
824 SmallVector<SDValue, 8> Ops(NumIntermediates);
825 for (unsigned i = 0; i != NumIntermediates; ++i) {
826 if (IntermediateVT.isVector()) {
827 // This does something sensible for scalable vectors - see the
828 // definition of EXTRACT_SUBVECTOR for further details.
829 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
830 Ops[i] =
831 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
832 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
833 } else {
834 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
835 DAG.getVectorIdxConstant(i, DL));
836 }
837 }
838
839 // Split the intermediate operands into legal parts.
840 if (NumParts == NumIntermediates) {
841 // If the register was not expanded, promote or copy the value,
842 // as appropriate.
843 for (unsigned i = 0; i != NumParts; ++i)
844 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
845 } else if (NumParts > 0) {
846 // If the intermediate type was expanded, split each the value into
847 // legal parts.
848 assert(NumIntermediates != 0 && "division by zero");
849 assert(NumParts % NumIntermediates == 0 &&
850 "Must expand into a divisible number of parts!");
851 unsigned Factor = NumParts / NumIntermediates;
852 for (unsigned i = 0; i != NumIntermediates; ++i)
853 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
854 CallConv);
855 }
856}
857
858static void failForInvalidBundles(const CallBase &I, StringRef Name,
859 ArrayRef<uint32_t> AllowedBundles) {
860 if (I.hasOperandBundlesOtherThan(AllowedBundles)) {
861 ListSeparator LS;
862 std::string Error;
864 for (unsigned i = 0, e = I.getNumOperandBundles(); i != e; ++i) {
865 OperandBundleUse U = I.getOperandBundleAt(i);
866 if (!is_contained(AllowedBundles, U.getTagID()))
867 OS << LS << U.getTagName();
868 }
870 Twine("cannot lower ", Name)
871 .concat(Twine(" with arbitrary operand bundles: ", Error)));
872 }
873}
874
876 EVT valuevt, std::optional<CallingConv::ID> CC)
877 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
878 RegCount(1, regs.size()), CallConv(CC) {}
879
881 const DataLayout &DL, Register Reg, Type *Ty,
882 std::optional<CallingConv::ID> CC) {
884
885 CallConv = CC;
886
887 for (EVT ValueVT : ValueVTs) {
888 unsigned NumRegs =
890 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
891 : TLI.getNumRegisters(Context, ValueVT);
892 MVT RegisterVT =
894 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
895 : TLI.getRegisterType(Context, ValueVT);
896 for (unsigned i = 0; i != NumRegs; ++i)
897 Regs.push_back(Reg + i);
898 RegVTs.push_back(RegisterVT);
899 RegCount.push_back(NumRegs);
900 Reg = Reg.id() + NumRegs;
901 }
902}
903
905 FunctionLoweringInfo &FuncInfo,
906 const SDLoc &dl, SDValue &Chain,
907 SDValue *Glue, const Value *V) const {
908 // A Value with type {} or [0 x %t] needs no registers.
909 if (ValueVTs.empty())
910 return SDValue();
911
913
914 // Assemble the legal parts into the final values.
917 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
918 // Copy the legal parts from the registers.
919 EVT ValueVT = ValueVTs[Value];
920 unsigned NumRegs = RegCount[Value];
921 MVT RegisterVT = isABIMangled()
922 ? TLI.getRegisterTypeForCallingConv(
923 *DAG.getContext(), *CallConv, RegVTs[Value])
924 : RegVTs[Value];
925
926 Parts.resize(NumRegs);
927 for (unsigned i = 0; i != NumRegs; ++i) {
928 SDValue P;
929 if (!Glue) {
930 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
931 } else {
932 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
933 *Glue = P.getValue(2);
934 }
935
936 Chain = P.getValue(1);
937 Parts[i] = P;
938
939 // If the source register was virtual and if we know something about it,
940 // add an assert node.
941 if (!Regs[Part + i].isVirtual() || !RegisterVT.isInteger())
942 continue;
943
945 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
946 if (!LOI)
947 continue;
948
949 unsigned RegSize = RegisterVT.getScalarSizeInBits();
950 unsigned NumSignBits = LOI->NumSignBits;
951 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
952
953 if (NumZeroBits == RegSize) {
954 // The current value is a zero.
955 // Explicitly express that as it would be easier for
956 // optimizations to kick in.
957 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
958 continue;
959 }
960
961 // FIXME: We capture more information than the dag can represent. For
962 // now, just use the tightest assertzext/assertsext possible.
963 bool isSExt;
964 EVT FromVT(MVT::Other);
965 if (NumZeroBits) {
966 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
967 isSExt = false;
968 } else if (NumSignBits > 1) {
969 FromVT =
970 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
971 isSExt = true;
972 } else {
973 continue;
974 }
975 // Add an assertion node.
976 assert(FromVT != MVT::Other);
977 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
978 RegisterVT, P, DAG.getValueType(FromVT));
979 }
980
981 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
982 RegisterVT, ValueVT, V, Chain, CallConv);
983 Part += NumRegs;
984 Parts.clear();
985 }
986
987 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
988}
989
991 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
992 const Value *V,
993 ISD::NodeType PreferredExtendType) const {
995 ISD::NodeType ExtendKind = PreferredExtendType;
996
997 // Get the list of the values's legal parts.
998 unsigned NumRegs = Regs.size();
999 SmallVector<SDValue, 8> Parts(NumRegs);
1000 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
1001 unsigned NumParts = RegCount[Value];
1002
1003 MVT RegisterVT = isABIMangled()
1004 ? TLI.getRegisterTypeForCallingConv(
1005 *DAG.getContext(), *CallConv, RegVTs[Value])
1006 : RegVTs[Value];
1007
1009 if (TLI.isZExtFree(peekThroughFreeze(Val), RegisterVT))
1011
1012 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
1013 NumParts, RegisterVT, V, CallConv, ExtendKind);
1014 Part += NumParts;
1015 }
1016
1017 // Copy the parts into the registers.
1018 SmallVector<SDValue, 8> Chains(NumRegs);
1019 for (unsigned i = 0; i != NumRegs; ++i) {
1020 SDValue Part;
1021 if (!Glue) {
1022 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
1023 } else {
1024 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
1025 *Glue = Part.getValue(1);
1026 }
1027
1028 Chains[i] = Part.getValue(0);
1029 }
1030
1031 if (NumRegs == 1 || Glue)
1032 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1033 // flagged to it. That is the CopyToReg nodes and the user are considered
1034 // a single scheduling unit. If we create a TokenFactor and return it as
1035 // chain, then the TokenFactor is both a predecessor (operand) of the
1036 // user as well as a successor (the TF operands are flagged to the user).
1037 // c1, f1 = CopyToReg
1038 // c2, f2 = CopyToReg
1039 // c3 = TokenFactor c1, c2
1040 // ...
1041 // = op c3, ..., f2
1042 Chain = Chains[NumRegs-1];
1043 else
1044 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1045}
1046
1048 unsigned MatchingIdx, const SDLoc &dl,
1049 SelectionDAG &DAG,
1050 std::vector<SDValue> &Ops) const {
1052
1053 InlineAsm::Flag Flag(Code, Regs.size());
1054 if (HasMatching)
1055 Flag.setMatchingOp(MatchingIdx);
1056 else if (!Regs.empty() && Regs.front().isVirtual()) {
1057 // Put the register class of the virtual registers in the flag word. That
1058 // way, later passes can recompute register class constraints for inline
1059 // assembly as well as normal instructions.
1060 // Don't do this for tied operands that can use the regclass information
1061 // from the def.
1063 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1064 Flag.setRegClass(RC->getID());
1065 }
1066
1067 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1068 Ops.push_back(Res);
1069
1070 if (Code == InlineAsm::Kind::Clobber) {
1071 // Clobbers should always have a 1:1 mapping with registers, and may
1072 // reference registers that have illegal (e.g. vector) types. Hence, we
1073 // shouldn't try to apply any sort of splitting logic to them.
1074 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1075 "No 1:1 mapping from clobbers to regs?");
1076 Register SP = TLI.getStackPointerRegisterToSaveRestore();
1077 (void)SP;
1078 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1079 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1080 assert(
1081 (Regs[I] != SP ||
1083 "If we clobbered the stack pointer, MFI should know about it.");
1084 }
1085 return;
1086 }
1087
1088 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1089 MVT RegisterVT = RegVTs[Value];
1090 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1091 RegisterVT);
1092 for (unsigned i = 0; i != NumRegs; ++i) {
1093 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1094 Register TheReg = Regs[Reg++];
1095 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1096 }
1097 }
1098}
1099
1103 unsigned I = 0;
1104 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1105 unsigned RegCount = std::get<0>(CountAndVT);
1106 MVT RegisterVT = std::get<1>(CountAndVT);
1107 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1108 for (unsigned E = I + RegCount; I != E; ++I)
1109 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1110 }
1111 return OutVec;
1112}
1113
1115 AssumptionCache *ac, const TargetLibraryInfo *li,
1116 const TargetTransformInfo &TTI) {
1117 BatchAA = aa;
1118 AC = ac;
1119 GFI = gfi;
1120 LibInfo = li;
1121 Context = DAG.getContext();
1122 LPadToCallSiteMap.clear();
1123 this->TTI = &TTI;
1124 SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1125 CanDescribeGlobalAddressInLocationList =
1126 canDescribeGlobalAddressInLocationList(DAG.getMachineFunction());
1127}
1128
1130 NodeMap.clear();
1131 UnusedArgNodeMap.clear();
1132 PendingLoads.clear();
1133 PendingExports.clear();
1134 PendingConstrainedFP.clear();
1135 PendingConstrainedFPStrict.clear();
1136 CurInst = nullptr;
1137 HasTailCall = false;
1138 SDNodeOrder = LowestSDNodeOrder;
1139 StatepointLowering.clear();
1140}
1141
1143 DanglingDebugInfoMap.clear();
1144}
1145
1146// Update DAG root to include dependencies on Pending chains.
1147SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1148 SDValue Root = DAG.getRoot();
1149
1150 if (Pending.empty())
1151 return Root;
1152
1153 // Add current root to PendingChains, unless we already indirectly
1154 // depend on it.
1155 if (Root.getOpcode() != ISD::EntryToken) {
1156 unsigned i = 0, e = Pending.size();
1157 for (; i != e; ++i) {
1158 assert(Pending[i].getNode()->getNumOperands() > 1);
1159 if (Pending[i].getNode()->getOperand(0) == Root)
1160 break; // Don't add the root if we already indirectly depend on it.
1161 }
1162
1163 if (i == e)
1164 Pending.push_back(Root);
1165 }
1166
1167 if (Pending.size() == 1)
1168 Root = Pending[0];
1169 else
1170 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1171
1172 DAG.setRoot(Root);
1173 Pending.clear();
1174 return Root;
1175}
1176
1180
1182 // If the new exception behavior differs from that of the pending
1183 // ones, chain up them and update the root.
1184 switch (EB) {
1187 // Floating-point exceptions produced by such operations are not intended
1188 // to be observed, so the sequence of these operations does not need to be
1189 // preserved.
1190 //
1191 // They however must not be mixed with the instructions that have strict
1192 // exception behavior. Placing an operation with 'ebIgnore' behavior between
1193 // 'ebStrict' operations could distort the observed exception behavior.
1194 if (!PendingConstrainedFPStrict.empty()) {
1195 assert(PendingConstrainedFP.empty());
1196 updateRoot(PendingConstrainedFPStrict);
1197 }
1198 break;
1200 // Floating-point exception produced by these operations may be observed, so
1201 // they must be correctly chained. If trapping on FP exceptions is
1202 // disabled, the exceptions can be observed only by functions that read
1203 // exception flags, like 'llvm.get_fpenv' or 'fetestexcept'. It means that
1204 // the order of operations is not significant between barriers.
1205 //
1206 // If trapping is enabled, each operation becomes an implicit observation
1207 // point, so the operations must be sequenced according their original
1208 // source order.
1209 if (!PendingConstrainedFP.empty()) {
1210 assert(PendingConstrainedFPStrict.empty());
1211 updateRoot(PendingConstrainedFP);
1212 }
1213 // TODO: Add support for trapping-enabled scenarios.
1214 }
1215 return DAG.getRoot();
1216}
1217
1219 // Chain up all pending constrained intrinsics together with all
1220 // pending loads, by simply appending them to PendingLoads and
1221 // then calling getMemoryRoot().
1222 PendingLoads.reserve(PendingLoads.size() +
1223 PendingConstrainedFP.size() +
1224 PendingConstrainedFPStrict.size());
1225 PendingLoads.append(PendingConstrainedFP.begin(),
1226 PendingConstrainedFP.end());
1227 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1228 PendingConstrainedFPStrict.end());
1229 PendingConstrainedFP.clear();
1230 PendingConstrainedFPStrict.clear();
1231 return getMemoryRoot();
1232}
1233
1235 // We need to emit pending fpexcept.strict constrained intrinsics,
1236 // so append them to the PendingExports list.
1237 PendingExports.append(PendingConstrainedFPStrict.begin(),
1238 PendingConstrainedFPStrict.end());
1239 PendingConstrainedFPStrict.clear();
1240 return updateRoot(PendingExports);
1241}
1242
1244 DILocalVariable *Variable,
1246 DebugLoc DL) {
1247 assert(Variable && "Missing variable");
1248
1249 // Check if address has undef value.
1250 if (!Address || isa<UndefValue>(Address) ||
1251 (Address->use_empty() && !isa<Argument>(Address))) {
1252 LLVM_DEBUG(
1253 dbgs()
1254 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1255 return;
1256 }
1257
1258 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1259
1260 SDValue &N = NodeMap[Address];
1261 if (!N.getNode() && isa<Argument>(Address))
1262 // Check unused arguments map.
1263 N = UnusedArgNodeMap[Address];
1264 SDDbgValue *SDV;
1265 if (N.getNode()) {
1266 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1267 Address = BCI->getOperand(0);
1268 // Parameters are handled specially.
1269 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1270 if (IsParameter && FINode) {
1271 // Byval parameter. We have a frame index at this point.
1272 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1273 /*IsIndirect*/ true, DL, SDNodeOrder);
1274 } else if (isa<Argument>(Address)) {
1275 // Address is an argument, so try to emit its dbg value using
1276 // virtual register info from the FuncInfo.ValueMap.
1277 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1278 FuncArgumentDbgValueKind::Declare, N);
1279 return;
1280 } else {
1281 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1282 true, DL, SDNodeOrder);
1283 }
1284 DAG.AddDbgValue(SDV, IsParameter);
1285 } else {
1286 // If Address is an argument then try to emit its dbg value using
1287 // virtual register info from the FuncInfo.ValueMap.
1288 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1289 FuncArgumentDbgValueKind::Declare, N)) {
1290 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1291 << " (could not emit func-arg dbg_value)\n");
1292 }
1293 }
1294}
1295
1297 // Add SDDbgValue nodes for any var locs here. Do so before updating
1298 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1299 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1300 // Add SDDbgValue nodes for any var locs here. Do so before updating
1301 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1302 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1303 It != End; ++It) {
1304 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1305 dropDanglingDebugInfo(Var, It->Expr);
1306 if (It->Values.isKillLocation(It->Expr)) {
1307 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1308 continue;
1309 }
1310 SmallVector<Value *> Values(It->Values.location_ops());
1311 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1312 It->Values.hasArgList())) {
1313 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1315 FnVarLocs->getDILocalVariable(It->VariableID),
1316 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1317 }
1318 }
1319 }
1320
1321 // We must skip DbgVariableRecords if they've already been processed above as
1322 // we have just emitted the debug values resulting from assignment tracking
1323 // analysis, making any existing DbgVariableRecords redundant (and probably
1324 // less correct). We still need to process DbgLabelRecords. This does sink
1325 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1326 // be important as it does so deterministcally and ordering between
1327 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1328 // printing).
1329 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1330 // Is there is any debug-info attached to this instruction, in the form of
1331 // DbgRecord non-instruction debug-info records.
1332 for (DbgRecord &DR : I.getDbgRecordRange()) {
1333 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1334 assert(DLR->getLabel() && "Missing label");
1335 SDDbgLabel *SDV =
1336 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1337 DAG.AddDbgLabel(SDV);
1338 continue;
1339 }
1340
1341 if (SkipDbgVariableRecords)
1342 continue;
1344 DILocalVariable *Variable = DVR.getVariable();
1347
1349 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1350 continue;
1351 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1352 << "\n");
1354 DVR.getDebugLoc());
1355 continue;
1356 }
1357
1358 // A DbgVariableRecord with no locations is a kill location.
1360 if (Values.empty()) {
1362 SDNodeOrder);
1363 continue;
1364 }
1365
1366 // A DbgVariableRecord with an undef or absent location is also a kill
1367 // location.
1368 if (llvm::any_of(Values,
1369 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1371 SDNodeOrder);
1372 continue;
1373 }
1374
1375 bool IsVariadic = DVR.hasArgList();
1376 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1377 SDNodeOrder, IsVariadic)) {
1378 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1379 DVR.getDebugLoc(), SDNodeOrder);
1380 }
1381 }
1382}
1383
1385 visitDbgInfo(I);
1386
1387 // Set up outgoing PHI node register values before emitting the terminator.
1388 if (I.isTerminator()) {
1389 HandlePHINodesInSuccessorBlocks(I.getParent());
1390 }
1391
1392 ++SDNodeOrder;
1393 CurInst = &I;
1394
1395 // Set inserted listener only if required.
1396 bool NodeInserted = false;
1397 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1398 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1399 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);
1400 if (PCSectionsMD || MMRA) {
1401 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1402 DAG, [&](SDNode *) { NodeInserted = true; });
1403 }
1404
1405 visit(I.getOpcode(), I);
1406
1407 if (!I.isTerminator() && !HasTailCall &&
1408 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1410
1411 // Handle metadata.
1412 if (PCSectionsMD || MMRA) {
1413 auto It = NodeMap.find(&I);
1414 if (It != NodeMap.end()) {
1415 if (PCSectionsMD)
1416 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1417 if (MMRA)
1418 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1419 } else if (NodeInserted) {
1420 // This should not happen; if it does, don't let it go unnoticed so we can
1421 // fix it. Relevant visit*() function is probably missing a setValue().
1422 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1423 << I.getModule()->getName() << "]\n";
1424 LLVM_DEBUG(I.dump());
1425 assert(false);
1426 }
1427 }
1428
1429 CurInst = nullptr;
1430}
1431
1432void SelectionDAGBuilder::visitPHI(const PHINode &) {
1433 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1434}
1435
1436void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1437 // Note: this doesn't use InstVisitor, because it has to work with
1438 // ConstantExpr's in addition to instructions.
1439 switch (Opcode) {
1440 default: llvm_unreachable("Unknown instruction type encountered!");
1441 // Build the switch statement using the Instruction.def file.
1442#define HANDLE_INST(NUM, OPCODE, CLASS) \
1443 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1444#include "llvm/IR/Instruction.def"
1445 }
1446}
1447
1449 DILocalVariable *Variable,
1450 DebugLoc DL, unsigned Order,
1453 // For variadic dbg_values we will now insert poison.
1454 // FIXME: We can potentially recover these!
1456 for (const Value *V : Values) {
1457 auto *Poison = PoisonValue::get(V->getType());
1458 Locs.push_back(SDDbgOperand::fromConst(Poison));
1459 }
1460 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1461 /*IsIndirect=*/false, DL, Order,
1462 /*IsVariadic=*/true);
1463 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1464 return true;
1465}
1466
1468 DILocalVariable *Var,
1469 DIExpression *Expr,
1470 bool IsVariadic, DebugLoc DL,
1471 unsigned Order) {
1472 if (IsVariadic) {
1473 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1474 return;
1475 }
1476 // TODO: Dangling debug info will eventually either be resolved or produce
1477 // a poison DBG_VALUE. However in the resolution case, a gap may appear
1478 // between the original dbg.value location and its resolved DBG_VALUE,
1479 // which we should ideally fill with an extra poison DBG_VALUE.
1480 assert(Values.size() == 1);
1481 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1482}
1483
1485 const DIExpression *Expr) {
1486 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1487 DIVariable *DanglingVariable = DDI.getVariable();
1488 DIExpression *DanglingExpr = DDI.getExpression();
1489 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1490 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1491 << printDDI(nullptr, DDI) << "\n");
1492 return true;
1493 }
1494 return false;
1495 };
1496
1497 for (auto &DDIMI : DanglingDebugInfoMap) {
1498 DanglingDebugInfoVector &DDIV = DDIMI.second;
1499
1500 // If debug info is to be dropped, run it through final checks to see
1501 // whether it can be salvaged.
1502 for (auto &DDI : DDIV)
1503 if (isMatchingDbgValue(DDI))
1504 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1505
1506 erase_if(DDIV, isMatchingDbgValue);
1507 }
1508}
1509
1510// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1511// generate the debug data structures now that we've seen its definition.
1513 SDValue Val) {
1514 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1515 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1516 return;
1517
1518 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1519 for (auto &DDI : DDIV) {
1520 DebugLoc DL = DDI.getDebugLoc();
1521 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1522 DILocalVariable *Variable = DDI.getVariable();
1523 DIExpression *Expr = DDI.getExpression();
1524 assert(Variable->isValidLocationForIntrinsic(DL) &&
1525 "Expected inlined-at fields to agree");
1526 SDDbgValue *SDV;
1527 if (Val.getNode()) {
1528 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1529 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1530 // we couldn't resolve it directly when examining the DbgValue intrinsic
1531 // in the first place we should not be more successful here). Unless we
1532 // have some test case that prove this to be correct we should avoid
1533 // calling EmitFuncArgumentDbgValue here.
1534 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1535 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1536 FuncArgumentDbgValueKind::Value, Val)) {
1537 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1538 << printDDI(V, DDI) << "\n");
1539 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1540 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1541 // inserted after the definition of Val when emitting the instructions
1542 // after ISel. An alternative could be to teach
1543 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1544 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1545 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1546 << ValSDNodeOrder << "\n");
1547 SDV = getDbgValue(Val, Variable, Expr, DL,
1548 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1549 DAG.AddDbgValue(SDV, false);
1550 } else
1551 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1552 << printDDI(V, DDI)
1553 << " in EmitFuncArgumentDbgValue\n");
1554 } else {
1555 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1556 << "\n");
1557 auto Poison = PoisonValue::get(V->getType());
1558 auto SDV =
1559 DAG.getConstantDbgValue(Variable, Expr, Poison, DL, DbgSDNodeOrder);
1560 DAG.AddDbgValue(SDV, false);
1561 }
1562 }
1563 DDIV.clear();
1564}
1565
1566/// If \p V is the address of a describable global, possibly displaced by a
1567/// constant, return the global and fold the displacement into location operand
1568/// \p OpIdx of \p Expr. The displacement rides along in the expression rather
1569/// than in the operand, so that it survives into a DBG_INSTR_REF.
1570static const GlobalValue *
1571getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx,
1572 const MachineFunction &MF) {
1573 const auto *C = dyn_cast<Constant>(V);
1574 if (!C)
1575 return nullptr;
1576 int64_t Offset;
1578 if (GV && Offset) {
1581 Expr = DIExpression::appendOpsToArg(Expr, Ops, OpIdx, /*StackValue=*/false);
1582 }
1583 return GV;
1584}
1585
1587 DanglingDebugInfo &DDI) {
1588 // TODO: For the variadic implementation, instead of only checking the fail
1589 // state of `handleDebugValue`, we need know specifically which values were
1590 // invalid, so that we attempt to salvage only those values when processing
1591 // a DIArgList.
1592 const Value *OrigV = V;
1593 DILocalVariable *Var = DDI.getVariable();
1594 DIExpression *Expr = DDI.getExpression();
1595 DebugLoc DL = DDI.getDebugLoc();
1596 unsigned SDOrder = DDI.getSDNodeOrder();
1597
1598 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1599 // that DW_OP_stack_value is desired.
1600 bool StackValue = true;
1601
1602 // handleDebugValue holds out for a register with the address of a global
1603 // that a location list could not name. With no such register forthcoming,
1604 // naming the global still beats dropping the location.
1605 auto HandleGlobalAddress = [&] {
1606 DIExpression *GVExpr = Expr;
1607 const GlobalValue *GV =
1608 getGlobalAddressDbgOperand(V, GVExpr, 0, DAG.getMachineFunction());
1609 if (!GV)
1610 return false;
1611 SDDbgValue *SDV = DAG.getDbgValueList(
1612 Var, GVExpr, SDDbgOperand::fromGlobalAddr(GV), /*Dependencies=*/{},
1613 /*IsIndirect=*/false, DL, SDOrder, /*IsVariadic=*/false);
1614 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1615 return true;
1616 };
1617
1618 // Can this Value can be encoded without any further work?
1619 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false) ||
1620 HandleGlobalAddress())
1621 return;
1622
1623 // Attempt to salvage back through as many instructions as possible. Bail if
1624 // a non-instruction is seen, such as a constant expression or global
1625 // variable. FIXME: Further work could recover those too.
1626 while (isa<Instruction>(V)) {
1627 const Instruction &VAsInst = *cast<const Instruction>(V);
1628 // Temporary "0", awaiting real implementation.
1630 SmallVector<Value *, 4> AdditionalValues;
1631 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1632 Expr->getNumLocationOperands(), Ops,
1633 AdditionalValues);
1634 // If we cannot salvage any further, and haven't yet found a suitable debug
1635 // expression, bail out.
1636 if (!V)
1637 break;
1638
1639 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1640 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1641 // here for variadic dbg_values, remove that condition.
1642 if (!AdditionalValues.empty())
1643 break;
1644
1645 // New value and expr now represent this debuginfo.
1647
1648 // Some kind of simplification occurred: check whether the operand of the
1649 // salvaged debug expression can be encoded in this DAG.
1650 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false) ||
1651 HandleGlobalAddress()) {
1652 LLVM_DEBUG(
1653 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1654 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1655 return;
1656 }
1657 }
1658
1659 // This was the final opportunity to salvage this debug information, and it
1660 // couldn't be done. Place a poison DBG_VALUE at this location to terminate
1661 // any earlier variable location.
1662 assert(OrigV && "V shouldn't be null");
1663 auto *Poison = PoisonValue::get(OrigV->getType());
1664 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Poison, DL, SDOrder);
1665 DAG.AddDbgValue(SDV, false);
1666 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1667 << printDDI(OrigV, DDI) << "\n");
1668}
1669
1671 DIExpression *Expr,
1672 DebugLoc DbgLoc,
1673 unsigned Order) {
1674 Value *Poison = PoisonValue::get(Type::getInt1Ty(*Context));
1677 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1678 /*IsVariadic*/ false);
1679}
1680
1682 DILocalVariable *Var,
1683 DIExpression *Expr, DebugLoc DbgLoc,
1684 unsigned Order, bool IsVariadic) {
1685 if (Values.empty())
1686 return true;
1687
1688 // Filter EntryValue locations out early.
1689 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1690 return true;
1691
1692 SmallVector<SDDbgOperand> LocationOps;
1693 SmallVector<SDNode *> Dependencies;
1694 for (const auto &[OpIdx, V] : enumerate(Values)) {
1695 // Constant value.
1698 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1699 continue;
1700 }
1701
1702 // Look through IntToPtr constants.
1703 if (auto *CE = dyn_cast<ConstantExpr>(V))
1704 if (CE->getOpcode() == Instruction::IntToPtr) {
1705 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1706 continue;
1707 }
1708
1709 // The address of a global is a link-time constant, and so is a constant
1710 // displacement from one. A global whose address cannot be described this
1711 // way falls through to be described by whatever materializes it instead.
1712 // So does one that a location list could not name, should the variable
1713 // need one; salvageUnresolvedDbgValue names it if nothing materializes it.
1714 if (CanDescribeGlobalAddressInLocationList)
1716 V, Expr, OpIdx, DAG.getMachineFunction())) {
1718 continue;
1719 }
1720
1721 // If the Value is a frame index, we can create a FrameIndex debug value
1722 // without relying on the DAG at all.
1723 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1724 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1725 if (SI != FuncInfo.StaticAllocaMap.end()) {
1726 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1727 continue;
1728 }
1729 }
1730
1731 // Do not use getValue() in here; we don't want to generate code at
1732 // this point if it hasn't been done yet.
1733 SDValue N = NodeMap[V];
1734 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1735 N = UnusedArgNodeMap[V];
1736
1737 if (N.getNode()) {
1738 // Only emit func arg dbg value for non-variadic dbg.values for now.
1739 if (!IsVariadic &&
1740 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1741 FuncArgumentDbgValueKind::Value, N))
1742 return true;
1743 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1744 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1745 // describe stack slot locations.
1746 //
1747 // Consider "int x = 0; int *px = &x;". There are two kinds of
1748 // interesting debug values here after optimization:
1749 //
1750 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1751 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1752 //
1753 // Both describe the direct values of their associated variables.
1754 Dependencies.push_back(N.getNode());
1755 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1756 continue;
1757 }
1758 LocationOps.emplace_back(
1759 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1760 continue;
1761 }
1762
1763 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1764 // Special rules apply for the first dbg.values of parameter variables in a
1765 // function. Identify them by the fact they reference Argument Values, that
1766 // they're parameters, and they are parameters of the current function. We
1767 // need to let them dangle until they get an SDNode.
1768 bool IsParamOfFunc =
1769 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1770 if (IsParamOfFunc)
1771 return false;
1772
1773 // The value is not used in this block yet (or it would have an SDNode).
1774 // We still want the value to appear for the user if possible -- if it has
1775 // an associated VReg, we can refer to that instead.
1776 auto VMI = FuncInfo.ValueMap.find(V);
1777 if (VMI != FuncInfo.ValueMap.end()) {
1778 Register Reg = VMI->second;
1779 // If this is a PHI node, it may be split up into several MI PHI nodes
1780 // (in FunctionLoweringInfo::set).
1781 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1782 V->getType(), std::nullopt);
1783 if (RFV.occupiesMultipleRegs()) {
1784 // FIXME: We could potentially support variadic dbg_values here.
1785 if (IsVariadic)
1786 return false;
1787 unsigned Offset = 0;
1788 unsigned BitsToDescribe = 0;
1789 if (auto VarSize = Var->getSizeInBits())
1790 BitsToDescribe = *VarSize;
1791 if (auto Fragment = Expr->getFragmentInfo())
1792 BitsToDescribe = Fragment->SizeInBits;
1793 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1794 // Bail out if all bits are described already.
1795 if (Offset >= BitsToDescribe)
1796 break;
1797 // TODO: handle scalable vectors.
1798 unsigned RegisterSize = RegAndSize.second;
1799 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1800 ? BitsToDescribe - Offset
1801 : RegisterSize;
1802 auto FragmentExpr = DIExpression::createFragmentExpression(
1803 Expr, Offset, FragmentSize);
1804 if (!FragmentExpr)
1805 continue;
1806 SDDbgValue *SDV = DAG.getVRegDbgValue(
1807 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);
1808 DAG.AddDbgValue(SDV, false);
1809 Offset += RegisterSize;
1810 }
1811 return true;
1812 }
1813 // We can use simple vreg locations for variadic dbg_values as well.
1814 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1815 continue;
1816 }
1817 // We failed to create a SDDbgOperand for V.
1818 return false;
1819 }
1820
1821 // We have created a SDDbgOperand for each Value in Values.
1822 assert(!LocationOps.empty());
1823 SDDbgValue *SDV =
1824 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1825 /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
1826 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1827 return true;
1828}
1829
1831 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1832 for (auto &Pair : DanglingDebugInfoMap)
1833 for (auto &DDI : Pair.second)
1834 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1836}
1837
1838/// getCopyFromRegs - If there was virtual register allocated for the value V
1839/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1841 auto It = FuncInfo.ValueMap.find(V);
1842 SDValue Result;
1843
1844 if (It != FuncInfo.ValueMap.end()) {
1845 Register InReg = It->second;
1846
1847 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1848 DAG.getDataLayout(), InReg, Ty,
1849 std::nullopt); // This is not an ABI copy.
1850 SDValue Chain = DAG.getEntryNode();
1851 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1852 V);
1853 resolveDanglingDebugInfo(V, Result);
1854 }
1855
1856 return Result;
1857}
1858
1859/// getValue - Return an SDValue for the given Value.
1861 // If we already have an SDValue for this value, use it. It's important
1862 // to do this first, so that we don't create a CopyFromReg if we already
1863 // have a regular SDValue.
1864 SDValue &N = NodeMap[V];
1865 if (N.getNode()) return N;
1866
1867 // If there's a virtual register allocated and initialized for this
1868 // value, use it.
1869 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1870 return copyFromReg;
1871
1872 // Otherwise create a new SDValue and remember it.
1873 SDValue Val = getValueImpl(V);
1874 NodeMap[V] = Val;
1876 return Val;
1877}
1878
1879void SelectionDAGBuilder::setValueToPoison(const Value *V, const SDLoc &dl) {
1880 if (V->getType()->isVoidTy())
1881 return;
1882
1883 SmallVector<EVT, 4> ValueVTs;
1884 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
1885 V->getType(), ValueVTs);
1886 if (ValueVTs.empty())
1887 return;
1888 setValue(V, DAG.getErrorMergeValues(ValueVTs, SDValue(), dl));
1889}
1890
1891/// getNonRegisterValue - Return an SDValue for the given Value, but
1892/// don't look in FuncInfo.ValueMap for a virtual register.
1894 // If we already have an SDValue for this value, use it.
1895 SDValue &N = NodeMap[V];
1896 if (N.getNode()) {
1897 if (isIntOrFPConstant(N)) {
1898 // Remove the debug location from the node as the node is about to be used
1899 // in a location which may differ from the original debug location. This
1900 // is relevant to Constant and ConstantFP nodes because they can appear
1901 // as constant expressions inside PHI nodes.
1902 N->setDebugLoc(DebugLoc());
1903 }
1904 return N;
1905 }
1906
1907 // Otherwise create a new SDValue and remember it.
1908 SDValue Val = getValueImpl(V);
1909 NodeMap[V] = Val;
1911 return Val;
1912}
1913
1914/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1915/// Create an SDValue for the given value.
1917 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1918
1919 if (const Constant *C = dyn_cast<Constant>(V)) {
1920 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1921
1922 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) {
1923 SDLoc DL = getCurSDLoc();
1924
1925 // DAG.getConstant() may attempt to legalise the vector constant which can
1926 // significantly change the combines applied to the DAG. To reduce the
1927 // divergence when enabling ConstantInt based vectors we try to construct
1928 // the DAG in the same way as shufflevector based splats. TODO: The
1929 // divergence sometimes leads to better optimisations. Ideally we should
1930 // prevent DAG.getConstant() from legalising too early but there are some
1931 // degradations preventing this.
1932 if (VT.isScalableVector())
1933 return DAG.getNode(
1934 ISD::SPLAT_VECTOR, DL, VT,
1935 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));
1936 if (VT.isFixedLengthVector())
1937 return DAG.getSplatBuildVector(
1938 VT, DL,
1939 DAG.getConstant(CI->getValue(), DL, VT.getVectorElementType()));
1940 return DAG.getConstant(*CI, DL, VT);
1941 }
1942
1943 if (const ConstantByte *CB = dyn_cast<ConstantByte>(C))
1944 return DAG.getConstant(CB->getValue(), getCurSDLoc(), VT);
1945
1946 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1947 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1948
1949 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
1950 return DAG.getNode(ISD::PtrAuthGlobalAddress, getCurSDLoc(), VT,
1951 getValue(CPA->getPointer()), getValue(CPA->getKey()),
1952 getValue(CPA->getAddrDiscriminator()),
1953 getValue(CPA->getDiscriminator()));
1954 }
1955
1957 return DAG.getConstant(0, getCurSDLoc(), VT);
1958
1959 if (match(C, m_VScale()))
1960 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1961
1962 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1963 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1964
1965 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1966 return isa<PoisonValue>(C) ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
1967
1968 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1969 visit(CE->getOpcode(), *CE);
1970 SDValue N1 = NodeMap[V];
1971 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1972 return N1;
1973 }
1974
1976 SmallVector<SDValue, 4> Constants;
1977 for (const Use &U : C->operands()) {
1978 SDNode *Val = getValue(U).getNode();
1979 // If the operand is an empty aggregate, there are no values.
1980 if (!Val) continue;
1981 // Add each leaf value from the operand to the Constants list
1982 // to form a flattened list of all the values.
1983 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1984 Constants.push_back(SDValue(Val, i));
1985 }
1986
1987 return DAG.getMergeValues(Constants, getCurSDLoc());
1988 }
1989
1990 if (const ConstantDataSequential *CDS =
1993 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1994 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1995 // Add each leaf value from the operand to the Constants list
1996 // to form a flattened list of all the values.
1997 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1998 Ops.push_back(SDValue(Val, i));
1999 }
2000
2001 if (isa<ArrayType>(CDS->getType()))
2002 return DAG.getMergeValues(Ops, getCurSDLoc());
2003 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);
2004 }
2005
2006 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
2008 "Unknown struct or array constant!");
2009
2010 SmallVector<EVT, 4> ValueVTs;
2011 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
2012 unsigned NumElts = ValueVTs.size();
2013 if (NumElts == 0)
2014 return SDValue(); // empty struct
2015 SmallVector<SDValue, 4> Constants(NumElts);
2016 for (unsigned i = 0; i != NumElts; ++i) {
2017 EVT EltVT = ValueVTs[i];
2018 if (isa<UndefValue>(C))
2019 Constants[i] = DAG.getUNDEF(EltVT);
2020 else if (EltVT.isFloatingPoint())
2021 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
2022 else
2023 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
2024 }
2025
2026 return DAG.getMergeValues(Constants, getCurSDLoc());
2027 }
2028
2029 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
2030 return DAG.getBlockAddress(BA, VT);
2031
2032 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
2033 return getValue(Equiv->getGlobalValue());
2034
2035 if (const auto *NC = dyn_cast<NoCFIValue>(C))
2036 return getValue(NC->getGlobalValue());
2037
2038 if (VT == MVT::aarch64svcount) {
2039 assert(C->isNullValue() && "Can only zero this target type!");
2040 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
2041 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
2042 }
2043
2044 if (VT.isRISCVVectorTuple()) {
2045 assert(C->isNullValue() && "Can only zero this target type!");
2046 return DAG.getNode(
2048 DAG.getNode(
2050 EVT::getVectorVT(*DAG.getContext(), MVT::i8,
2051 VT.getSizeInBits().getKnownMinValue() / 8, true),
2052 DAG.getConstant(0, getCurSDLoc(), MVT::getIntegerVT(8))));
2053 }
2054
2055 if (VT == MVT::externref || VT == MVT::funcref) {
2056 assert(C->isNullValue() && "Can only zero this target type!");
2057 // The zero value of a WebAssembly reference type is the null reference,
2058 // materialized with ref.null.
2059 Intrinsic::ID IID = VT == MVT::externref ? Intrinsic::wasm_ref_null_extern
2060 : Intrinsic::wasm_ref_null_func;
2061 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VT,
2062 DAG.getTargetConstant(IID, getCurSDLoc(), MVT::i32));
2063 }
2064
2065 VectorType *VecTy = cast<VectorType>(V->getType());
2066
2067 // Now that we know the number and type of the elements, get that number of
2068 // elements into the Ops array based on what kind of constant it is.
2069 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
2071 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
2072 for (unsigned i = 0; i != NumElements; ++i)
2073 Ops.push_back(getValue(CV->getOperand(i)));
2074
2075 return DAG.getBuildVector(VT, getCurSDLoc(), Ops);
2076 }
2077
2079 EVT EltVT =
2080 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
2081
2082 SDValue Op;
2083 if (EltVT.isFloatingPoint())
2084 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
2085 else
2086 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
2087
2088 return DAG.getSplat(VT, getCurSDLoc(), Op);
2089 }
2090
2091 llvm_unreachable("Unknown vector constant");
2092 }
2093
2094 // If this is a static alloca, generate it as the frameindex instead of
2095 // computation.
2096 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
2097 auto SI = FuncInfo.StaticAllocaMap.find(AI);
2098 if (SI != FuncInfo.StaticAllocaMap.end())
2099 return DAG.getFrameIndex(
2100 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
2101 }
2102
2103 // If this is an instruction which fast-isel has deferred, select it now.
2104 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
2105 Register InReg = FuncInfo.InitializeRegForValue(Inst);
2106 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
2107 Inst->getType(), std::nullopt);
2108 SDValue Chain = DAG.getEntryNode();
2109 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
2110 }
2111
2112 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
2113 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
2114
2115 if (const auto *BB = dyn_cast<BasicBlock>(V))
2116 return DAG.getBasicBlock(FuncInfo.getMBB(BB));
2117
2118 llvm_unreachable("Can't get register for value!");
2119}
2120
2121void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
2123 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
2124 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
2125 bool IsSEH = isAsynchronousEHPersonality(Pers);
2126 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
2127 if (IsSEH) {
2128 // For SEH, EHCont Guard needs to know that this catchpad is a target.
2129 CatchPadMBB->setIsEHContTarget(true);
2131 } else
2132 CatchPadMBB->setIsEHScopeEntry();
2133 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
2134 if (IsMSVCCXX || IsCoreCLR)
2135 CatchPadMBB->setIsEHFuncletEntry();
2136}
2137
2138void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
2139 // Update machine-CFG edge.
2140 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());
2141 FuncInfo.MBB->addSuccessor(TargetMBB);
2142
2143 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
2144 bool IsSEH = isAsynchronousEHPersonality(Pers);
2145 if (IsSEH) {
2146 // If this is not a fall-through branch or optimizations are switched off,
2147 // emit the branch.
2148 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
2149 TM.getOptLevel() == CodeGenOptLevel::None)
2150 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2151 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
2152 return;
2153 }
2154
2155 // For non-SEH, EHCont Guard needs to know that this catchret is a target.
2156 TargetMBB->setIsEHContTarget(true);
2157 DAG.getMachineFunction().setHasEHContTarget(true);
2158
2159 // Figure out the funclet membership for the catchret's successor.
2160 // This will be used by the FuncletLayout pass to determine how to order the
2161 // BB's.
2162 // A 'catchret' returns to the outer scope's color.
2163 Value *ParentPad = I.getCatchSwitchParentPad();
2164 const BasicBlock *SuccessorColor;
2165 if (isa<ConstantTokenNone>(ParentPad))
2166 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2167 else
2168 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
2169 assert(SuccessorColor && "No parent funclet for catchret!");
2170 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(SuccessorColor);
2171 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2172
2173 // Create the terminator node.
2174 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
2175 getControlRoot(), DAG.getBasicBlock(TargetMBB),
2176 DAG.getBasicBlock(SuccessorColorMBB));
2177 DAG.setRoot(Ret);
2178}
2179
2180void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2181 // Don't emit any special code for the cleanuppad instruction. It just marks
2182 // the start of an EH scope/funclet.
2183 FuncInfo.MBB->setIsEHScopeEntry();
2184 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
2185 if (Pers != EHPersonality::Wasm_CXX && Pers != EHPersonality::Wasm_D) {
2186 FuncInfo.MBB->setIsEHFuncletEntry();
2187 FuncInfo.MBB->setIsCleanupFuncletEntry();
2188 }
2189}
2190
2191/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2192/// many places it could ultimately go. In the IR, we have a single unwind
2193/// destination, but in the machine CFG, we enumerate all the possible blocks.
2194/// This function skips over imaginary basic blocks that hold catchswitch
2195/// instructions, and finds all the "real" machine
2196/// basic block destinations. As those destinations may not be successors of
2197/// EHPadBB, here we also calculate the edge probability to those destinations.
2198/// The passed-in Prob is the edge probability to EHPadBB.
2200 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2201 BranchProbability Prob,
2202 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2203 &UnwindDests) {
2204 EHPersonality Personality =
2206 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2207 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2208 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2209 bool IsWasmD = Personality == EHPersonality::Wasm_D;
2210 bool IsSEH = isAsynchronousEHPersonality(Personality);
2211
2212 while (EHPadBB) {
2214 BasicBlock *NewEHPadBB = nullptr;
2215 if (isa<LandingPadInst>(Pad)) {
2216 // Stop on landingpads. They are not funclets.
2217 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2218 break;
2219 } else if (isa<CleanupPadInst>(Pad)) {
2220 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2221 // personalities except Wasm. And in Wasm this becomes a catch_all(_ref),
2222 // which always catches an exception.
2223 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2224 UnwindDests.back().first->setIsEHScopeEntry();
2225 // In Wasm, EH scopes are not funclets
2226 if (!IsWasmCXX && !IsWasmD)
2227 UnwindDests.back().first->setIsEHFuncletEntry();
2228 break;
2229 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2230 // Add the catchpad handlers to the possible destinations.
2231 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2232 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2233 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2234 if (IsMSVCCXX || IsCoreCLR)
2235 UnwindDests.back().first->setIsEHFuncletEntry();
2236 if (!IsSEH)
2237 UnwindDests.back().first->setIsEHScopeEntry();
2238 }
2239 NewEHPadBB = CatchSwitch->getUnwindDest();
2240 } else {
2241 continue;
2242 }
2243
2244 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2245 if (BPI && NewEHPadBB)
2246 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2247 EHPadBB = NewEHPadBB;
2248 }
2249}
2250
2251void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2252 // Update successor info.
2254 auto UnwindDest = I.getUnwindDest();
2255 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2256 BranchProbability UnwindDestProb =
2257 (BPI && UnwindDest)
2258 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2260 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2261 for (auto &UnwindDest : UnwindDests) {
2262 UnwindDest.first->setIsEHPad();
2263 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2264 }
2265 FuncInfo.MBB->normalizeSuccProbs();
2266
2267 // Create the terminator node.
2268 MachineBasicBlock *CleanupPadMBB =
2269 FuncInfo.getMBB(I.getCleanupPad()->getParent());
2270 SDValue Ret = DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other,
2271 getControlRoot(), DAG.getBasicBlock(CleanupPadMBB));
2272 DAG.setRoot(Ret);
2273}
2274
2275void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2276 report_fatal_error("visitCatchSwitch not yet implemented!");
2277}
2278
2279void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2280 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2281 auto &DL = DAG.getDataLayout();
2282 SDValue Chain = getControlRoot();
2285
2286 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2287 // lower
2288 //
2289 // %val = call <ty> @llvm.experimental.deoptimize()
2290 // ret <ty> %val
2291 //
2292 // differently.
2293 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2295 return;
2296 }
2297
2298 if (!FuncInfo.CanLowerReturn) {
2299 Register DemoteReg = FuncInfo.DemoteRegister;
2300
2301 // Emit a store of the return value through the virtual register.
2302 // Leave Outs empty so that LowerReturn won't try to load return
2303 // registers the usual way.
2304 MVT PtrValueVT = TLI.getPointerTy(DL, DL.getAllocaAddrSpace());
2305 SDValue RetPtr =
2306 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVT);
2307 Type *RetTy = I.getOperand(0)->getType();
2308 Align BaseAlign = DL.getPrefTypeAlign(RetTy);
2309 RetPtr =
2310 TLI.annotateStackObjectPointer(RetPtr, DAG, getCurSDLoc(), BaseAlign);
2311 SDValue RetOp = getValue(I.getOperand(0));
2312
2313 SmallVector<EVT, 4> ValueVTs, MemVTs;
2314 SmallVector<uint64_t, 4> Offsets;
2315 ComputeValueVTs(TLI, DL, RetTy, ValueVTs, &MemVTs, &Offsets, 0);
2316 unsigned NumValues = ValueVTs.size();
2317
2318 SmallVector<SDValue, 4> Chains(NumValues);
2319 for (unsigned i = 0; i != NumValues; ++i) {
2320 // An aggregate return value cannot wrap around the address space, so
2321 // offsets to its parts don't wrap either.
2322 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
2323 TypeSize::getFixed(Offsets[i]));
2324
2325 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2326 if (MemVTs[i] != ValueVTs[i])
2327 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2328 Chains[i] = DAG.getStore(
2329 Chain, getCurSDLoc(), Val,
2330 // FIXME: better loc info would be nice.
2331 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
2332 commonAlignment(BaseAlign, Offsets[i]));
2333 }
2334
2335 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
2336 MVT::Other, Chains);
2337 } else if (I.getNumOperands() != 0) {
2339 ComputeValueTypes(DL, I.getOperand(0)->getType(), Types);
2340 unsigned NumValues = Types.size();
2341 if (NumValues) {
2342 SDValue RetOp = getValue(I.getOperand(0));
2343
2344 const Function *F = I.getParent()->getParent();
2345
2347 I.getOperand(0)->getType(), F->getCallingConv(),
2348 /*IsVarArg*/ false, DL);
2349
2351 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2353 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2355
2356 LLVMContext &Context = F->getContext();
2357 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2358
2359 for (unsigned j = 0; j != NumValues; ++j) {
2360 EVT VT = TLI.getValueType(DL, Types[j]);
2361
2362 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2364
2365 CallingConv::ID CC = F->getCallingConv();
2366
2368 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2371 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2372 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2373
2374 // 'inreg' on function refers to return value
2375 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2376 if (RetInReg)
2377 Flags.setInReg();
2378
2379 if (I.getOperand(0)->getType()->isPointerTy()) {
2380 Flags.setPointer();
2381 Flags.setPointerAddrSpace(
2382 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2383 }
2384
2385 if (NeedsRegBlock) {
2386 Flags.setInConsecutiveRegs();
2387 if (j == NumValues - 1)
2388 Flags.setInConsecutiveRegsLast();
2389 }
2390
2391 // Propagate extension type if any
2393 Flags.setSExt();
2394 else if (ExtendKind == ISD::ZERO_EXTEND)
2395 Flags.setZExt();
2396 else if (F->getAttributes().hasRetAttr(Attribute::NoExt))
2397 Flags.setNoExt();
2398
2399 for (unsigned i = 0; i < NumParts; ++i) {
2400 Outs.push_back(ISD::OutputArg(Flags,
2401 Parts[i].getValueType().getSimpleVT(),
2402 VT, Types[j], 0, 0));
2403 OutVals.push_back(Parts[i]);
2404 }
2405 }
2406 }
2407 }
2408
2409 // Push in swifterror virtual register as the last element of Outs. This makes
2410 // sure swifterror virtual register will be returned in the swifterror
2411 // physical register.
2412 const Function *F = I.getParent()->getParent();
2413 if (TLI.supportSwiftError() &&
2414 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2415 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2416 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
2417 Flags.setSwiftError();
2418 Outs.push_back(ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
2419 /*argvt=*/EVT(TLI.getPointerTy(DL)),
2420 PointerType::getUnqual(*DAG.getContext()),
2421 /*origidx=*/1, /*partOffs=*/0));
2422 // Create SDNode for the swifterror virtual register.
2423 OutVals.push_back(
2424 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
2425 &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
2426 EVT(TLI.getPointerTy(DL))));
2427 }
2428
2429 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2430 CallingConv::ID CallConv =
2431 DAG.getMachineFunction().getFunction().getCallingConv();
2432 Chain = DAG.getTargetLoweringInfo().LowerReturn(
2433 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2434
2435 // Verify that the target's LowerReturn behaved as expected.
2436 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2437 "LowerReturn didn't return a valid chain!");
2438
2439 // Update the DAG with the new chain value resulting from return lowering.
2440 DAG.setRoot(Chain);
2441}
2442
2443/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2444/// created for it, emit nodes to copy the value into the virtual
2445/// registers.
2447 // Skip empty types
2448 if (V->getType()->isEmptyTy())
2449 return;
2450
2451 auto VMI = FuncInfo.ValueMap.find(V);
2452 if (VMI != FuncInfo.ValueMap.end()) {
2453 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2454 "Unused value assigned virtual registers!");
2455 CopyValueToVirtualRegister(V, VMI->second);
2456 }
2457}
2458
2459/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2460/// the current basic block, add it to ValueMap now so that we'll get a
2461/// CopyTo/FromReg.
2463 // No need to export constants.
2464 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2465
2466 // Already exported?
2467 if (FuncInfo.isExportedInst(V)) return;
2468
2469 Register Reg = FuncInfo.InitializeRegForValue(V);
2471}
2472
2474 const BasicBlock *FromBB) {
2475 // The operands of the setcc have to be in this block. We don't know
2476 // how to export them from some other block.
2477 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2478 // Can export from current BB.
2479 if (VI->getParent() == FromBB)
2480 return true;
2481
2482 // Is already exported, noop.
2483 return FuncInfo.isExportedInst(V);
2484 }
2485
2486 // If this is an argument, we can export it if the BB is the entry block or
2487 // if it is already exported.
2488 if (isa<Argument>(V)) {
2489 if (FromBB->isEntryBlock())
2490 return true;
2491
2492 // Otherwise, can only export this if it is already exported.
2493 return FuncInfo.isExportedInst(V);
2494 }
2495
2496 // Otherwise, constants can always be exported.
2497 return true;
2498}
2499
2500/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2502SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2503 const MachineBasicBlock *Dst) const {
2505 const BasicBlock *SrcBB = Src->getBasicBlock();
2506 const BasicBlock *DstBB = Dst->getBasicBlock();
2507 if (!BPI) {
2508 // If BPI is not available, set the default probability as 1 / N, where N is
2509 // the number of successors.
2510 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2511 return BranchProbability(1, SuccSize);
2512 }
2513 return BPI->getEdgeProbability(SrcBB, DstBB);
2514}
2515
2516void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2517 MachineBasicBlock *Dst,
2518 BranchProbability Prob) {
2519 if (!FuncInfo.BPI)
2520 Src->addSuccessorWithoutProb(Dst);
2521 else {
2522 if (Prob.isUnknown())
2523 Prob = getEdgeProbability(Src, Dst);
2524 Src->addSuccessor(Dst, Prob);
2525 }
2526}
2527
2528static bool InBlock(const Value *V, const BasicBlock *BB) {
2529 if (const Instruction *I = dyn_cast<Instruction>(V))
2530 return I->getParent() == BB;
2531 return true;
2532}
2533
2534/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2535/// This function emits a branch and is used at the leaves of an OR or an
2536/// AND operator tree.
2537void
2540 MachineBasicBlock *FBB,
2541 MachineBasicBlock *CurBB,
2542 MachineBasicBlock *SwitchBB,
2543 BranchProbability TProb,
2544 BranchProbability FProb,
2545 bool InvertCond) {
2546 const BasicBlock *BB = CurBB->getBasicBlock();
2547
2548 // If the leaf of the tree is a comparison, merge the condition into
2549 // the caseblock.
2550 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2551 // The operands of the cmp have to be in this block. We don't know
2552 // how to export them from some other block. If this is the first block
2553 // of the sequence, no exporting is needed.
2554 if (CurBB == SwitchBB ||
2555 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2556 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2557 ISD::CondCode Condition;
2558 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2559 ICmpInst::Predicate Pred =
2560 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2561 Condition = getICmpCondCode(Pred);
2562 } else {
2563 const FCmpInst *FC = cast<FCmpInst>(Cond);
2564 FCmpInst::Predicate Pred =
2565 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2566 Condition = getFCmpCondCode(Pred);
2567 if (FC->hasNoNaNs() ||
2568 (isKnownNeverNaN(FC->getOperand(0),
2569 SimplifyQuery(DAG.getDataLayout(), FC)) &&
2570 isKnownNeverNaN(FC->getOperand(1),
2571 SimplifyQuery(DAG.getDataLayout(), FC))))
2572 Condition = getFCmpCodeWithoutNaN(Condition);
2573 }
2574
2575 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2576 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2577 SL->SwitchCases.push_back(CB);
2578 return;
2579 }
2580 }
2581
2582 // Create a CaseBlock record representing this branch.
2583 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2584 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2585 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2586 SL->SwitchCases.push_back(CB);
2587}
2588
2589// Collect dependencies on V recursively. This is used for the cost analysis in
2590// `shouldKeepJumpConditionsTogether`.
2594 unsigned Depth = 0) {
2595 // Return false if we have an incomplete count.
2597 return false;
2598
2599 auto *I = dyn_cast<Instruction>(V);
2600 if (I == nullptr)
2601 return true;
2602
2603 if (Necessary != nullptr) {
2604 // This instruction is necessary for the other side of the condition so
2605 // don't count it.
2606 if (Necessary->contains(I))
2607 return true;
2608 }
2609
2610 // Already added this dep.
2611 if (!Deps->try_emplace(I, false).second)
2612 return true;
2613
2614 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2615 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2616 Depth + 1))
2617 return false;
2618 return true;
2619}
2620
2623 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2625 if (Params.BaseCost < 0)
2626 return false;
2627
2628 // Baseline cost.
2629 InstructionCost CostThresh = Params.BaseCost;
2630
2631 BranchProbabilityInfo *BPI = nullptr;
2632 if (Params.LikelyBias || Params.UnlikelyBias)
2633 BPI = FuncInfo.BPI;
2634 if (BPI != nullptr) {
2635 // See if we are either likely to get an early out or compute both lhs/rhs
2636 // of the condition.
2637 BasicBlock *IfTrue = I.getSuccessor(0);
2638 BasicBlock *IfFalse = I.getSuccessor(1);
2640 std::swap(IfTrue, IfFalse);
2641
2642 std::optional<bool> Likely;
2643 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2644 Likely = true;
2645 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2646 Likely = false;
2647
2648 if (Likely) {
2649 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2650 // Its likely we will have to compute both lhs and rhs of condition
2651 CostThresh += Params.LikelyBias;
2652 else {
2653 if (Params.UnlikelyBias < 0)
2654 return false;
2655 // Its likely we will get an early out.
2656 CostThresh -= Params.UnlikelyBias;
2657 }
2658 }
2659 }
2660
2661 if (CostThresh <= 0)
2662 return false;
2663
2664 // Collect "all" instructions that lhs condition is dependent on.
2665 // Use map for stable iteration (to avoid non-determanism of iteration of
2666 // SmallPtrSet). The `bool` value is just a dummy.
2668 collectInstructionDeps(&LhsDeps, Lhs);
2669 // Collect "all" instructions that rhs condition is dependent on AND are
2670 // dependencies of lhs. This gives us an estimate on which instructions we
2671 // stand to save by splitting the condition.
2672 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2673 return false;
2674 // Add the compare instruction itself unless its a dependency on the LHS.
2675 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2676 if (!LhsDeps.contains(RhsI))
2677 RhsDeps.try_emplace(RhsI, false);
2678
2679 InstructionCost CostOfIncluding = 0;
2680 // See if this instruction will need to computed independently of whether RHS
2681 // is.
2682 Value *BrCond = I.getCondition();
2683 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2684 for (const auto *U : Ins->users()) {
2685 // If user is independent of RHS calculation we don't need to count it.
2686 if (auto *UIns = dyn_cast<Instruction>(U))
2687 if (UIns != BrCond && !RhsDeps.contains(UIns))
2688 return false;
2689 }
2690 return true;
2691 };
2692
2693 // Prune instructions from RHS Deps that are dependencies of unrelated
2694 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2695 // arbitrary and just meant to cap the how much time we spend in the pruning
2696 // loop. Its highly unlikely to come into affect.
2697 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2698 // Stop after a certain point. No incorrectness from including too many
2699 // instructions.
2700 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2701 const Instruction *ToDrop = nullptr;
2702 for (const auto &InsPair : RhsDeps) {
2703 if (!ShouldCountInsn(InsPair.first)) {
2704 ToDrop = InsPair.first;
2705 break;
2706 }
2707 }
2708 if (ToDrop == nullptr)
2709 break;
2710 RhsDeps.erase(ToDrop);
2711 }
2712
2713 for (const auto &InsPair : RhsDeps) {
2714 // Finally accumulate latency that we can only attribute to computing the
2715 // RHS condition. Use latency because we are essentially trying to calculate
2716 // the cost of the dependency chain.
2717 // Possible TODO: We could try to estimate ILP and make this more precise.
2718 CostOfIncluding += TTI->getInstructionCost(
2719 InsPair.first, TargetTransformInfo::TCK_Latency);
2720
2721 if (CostOfIncluding > CostThresh)
2722 return false;
2723 }
2724 return true;
2725}
2726
2729 MachineBasicBlock *FBB,
2730 MachineBasicBlock *CurBB,
2731 MachineBasicBlock *SwitchBB,
2733 BranchProbability TProb,
2734 BranchProbability FProb,
2735 bool InvertCond) {
2736 // Skip over not part of the tree and remember to invert op and operands at
2737 // next level.
2738 Value *NotCond;
2739 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2740 InBlock(NotCond, CurBB->getBasicBlock())) {
2741 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2742 !InvertCond);
2743 return;
2744 }
2745
2747 const Value *BOpOp0, *BOpOp1;
2748 // Compute the effective opcode for Cond, taking into account whether it needs
2749 // to be inverted, e.g.
2750 // and (not (or A, B)), C
2751 // gets lowered as
2752 // and (and (not A, not B), C)
2754 if (BOp) {
2755 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2756 ? Instruction::And
2757 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2758 ? Instruction::Or
2760 if (InvertCond) {
2761 if (BOpc == Instruction::And)
2762 BOpc = Instruction::Or;
2763 else if (BOpc == Instruction::Or)
2764 BOpc = Instruction::And;
2765 }
2766 }
2767
2768 // If this node is not part of the or/and tree, emit it as a branch.
2769 // Note that all nodes in the tree should have same opcode.
2770 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2771 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2772 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2773 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2774 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2775 TProb, FProb, InvertCond);
2776 return;
2777 }
2778
2779 // Create TmpBB after CurBB.
2780 MachineFunction::iterator BBI(CurBB);
2781 MachineFunction &MF = DAG.getMachineFunction();
2783 CurBB->getParent()->insert(++BBI, TmpBB);
2784
2785 if (Opc == Instruction::Or) {
2786 // Codegen X | Y as:
2787 // BB1:
2788 // jmp_if_X TBB
2789 // jmp TmpBB
2790 // TmpBB:
2791 // jmp_if_Y TBB
2792 // jmp FBB
2793 //
2794
2795 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2796 // The requirement is that
2797 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2798 // = TrueProb for original BB.
2799 // Assuming the original probabilities are A and B, one choice is to set
2800 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2801 // A/(1+B) and 2B/(1+B). This choice assumes that
2802 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2803 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2804 // TmpBB, but the math is more complicated.
2805
2806 auto NewTrueProb = TProb / 2;
2807 auto NewFalseProb = TProb / 2 + FProb;
2808 // Emit the LHS condition.
2809 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2810 NewFalseProb, InvertCond);
2811
2812 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2813 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2815 // Emit the RHS condition into TmpBB.
2816 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2817 Probs[1], InvertCond);
2818 } else {
2819 assert(Opc == Instruction::And && "Unknown merge op!");
2820 // Codegen X & Y as:
2821 // BB1:
2822 // jmp_if_X TmpBB
2823 // jmp FBB
2824 // TmpBB:
2825 // jmp_if_Y TBB
2826 // jmp FBB
2827 //
2828 // This requires creation of TmpBB after CurBB.
2829
2830 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2831 // The requirement is that
2832 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2833 // = FalseProb for original BB.
2834 // Assuming the original probabilities are A and B, one choice is to set
2835 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2836 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2837 // TrueProb for BB1 * FalseProb for TmpBB.
2838
2839 auto NewTrueProb = TProb + FProb / 2;
2840 auto NewFalseProb = FProb / 2;
2841 // Emit the LHS condition.
2842 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2843 NewFalseProb, InvertCond);
2844
2845 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2846 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2848 // Emit the RHS condition into TmpBB.
2849 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2850 Probs[1], InvertCond);
2851 }
2852}
2853
2854/// If the set of cases should be emitted as a series of branches, return true.
2855/// If we should emit this as a bunch of and/or'd together conditions, return
2856/// false.
2857bool
2858SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2859 if (Cases.size() != 2) return true;
2860
2861 // If this is two comparisons of the same values or'd or and'd together, they
2862 // will get folded into a single comparison, so don't emit two blocks.
2863 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2864 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2865 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2866 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2867 return false;
2868 }
2869
2870 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2871 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2872 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2873 Cases[0].CC == Cases[1].CC &&
2874 isa<Constant>(Cases[0].CmpRHS) &&
2875 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2876 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2877 return false;
2878 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2879 return false;
2880 }
2881
2882 return true;
2883}
2884
2885void SelectionDAGBuilder::visitUncondBr(const UncondBrInst &I) {
2887
2888 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2889
2890 // Update machine-CFG edges.
2891 BrMBB->addSuccessor(Succ0MBB);
2892
2893 // If this is not a fall-through branch or optimizations are switched off,
2894 // emit the branch.
2895 if (Succ0MBB != NextBlock(BrMBB) ||
2897 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2898 DAG.getBasicBlock(Succ0MBB));
2899 setValue(&I, Br);
2900 DAG.setRoot(Br);
2901 }
2902}
2903
2904void SelectionDAGBuilder::visitCondBr(const CondBrInst &I) {
2905 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2906
2907 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2908
2909 // If this condition is one of the special cases we handle, do special stuff
2910 // now.
2911 const Value *CondVal = I.getCondition();
2912 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(I.getSuccessor(1));
2913
2914 // If this is a series of conditions that are or'd or and'd together, emit
2915 // this as a sequence of branches instead of setcc's with and/or operations.
2916 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2917 // unpredictable branches, and vector extracts because those jumps are likely
2918 // expensive for any target), this should improve performance.
2919 // For example, instead of something like:
2920 // cmp A, B
2921 // C = seteq
2922 // cmp D, E
2923 // F = setle
2924 // or C, F
2925 // jnz foo
2926 // Emit:
2927 // cmp A, B
2928 // je foo
2929 // cmp D, E
2930 // jle foo
2931 bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
2932 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2933 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2934 BOp->hasOneUse() && !IsUnpredictable) {
2935 Value *Vec;
2936 const Value *BOp0, *BOp1;
2938 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2939 Opcode = Instruction::And;
2940 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2941 Opcode = Instruction::Or;
2942
2943 if (Opcode &&
2944 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2945 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2947 FuncInfo, I, Opcode, BOp0, BOp1,
2948 DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2949 Opcode, BOp0, BOp1, FuncInfo.Fn))) {
2950 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2951 getEdgeProbability(BrMBB, Succ0MBB),
2952 getEdgeProbability(BrMBB, Succ1MBB),
2953 /*InvertCond=*/false);
2954 // If the compares in later blocks need to use values not currently
2955 // exported from this block, export them now. This block should always
2956 // be the first entry.
2957 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2958
2959 // Allow some cases to be rejected.
2960 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2961 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2962 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2963 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2964 }
2965
2966 // Emit the branch for this block.
2967 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2968 SL->SwitchCases.erase(SL->SwitchCases.begin());
2969 return;
2970 }
2971
2972 // Okay, we decided not to do this, remove any inserted MBB's and clear
2973 // SwitchCases.
2974 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2975 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2976
2977 SL->SwitchCases.clear();
2978 }
2979 }
2980
2981 // Create a CaseBlock record representing this branch.
2982 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2983 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2985 IsUnpredictable);
2986
2987 // Use visitSwitchCase to actually insert the fast branch sequence for this
2988 // cond branch.
2989 visitSwitchCase(CB, BrMBB);
2990}
2991
2992/// visitSwitchCase - Emits the necessary code to represent a single node in
2993/// the binary search tree resulting from lowering a switch instruction.
2995 MachineBasicBlock *SwitchBB) {
2996 SDValue Cond;
2997 SDValue CondLHS = getValue(CB.CmpLHS);
2998 SDLoc dl = CB.DL;
2999
3000 if (CB.CC == ISD::SETTRUE) {
3001 // Branch or fall through to TrueBB.
3002 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
3003 SwitchBB->normalizeSuccProbs();
3004 if (CB.TrueBB != NextBlock(SwitchBB)) {
3005 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
3006 DAG.getBasicBlock(CB.TrueBB)));
3007 }
3008 return;
3009 }
3010
3011 auto &TLI = DAG.getTargetLoweringInfo();
3012 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
3013
3014 // Build the setcc now.
3015 if (!CB.CmpMHS) {
3016 // Fold "(X == true)" to X and "(X == false)" to !X to
3017 // handle common cases produced by branch lowering.
3018 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
3019 CB.CC == ISD::SETEQ)
3020 Cond = CondLHS;
3021 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
3022 CB.CC == ISD::SETEQ) {
3023 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
3024 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
3025 } else {
3026 SDValue CondRHS = getValue(CB.CmpRHS);
3027
3028 // If a pointer's DAG type is larger than its memory type then the DAG
3029 // values are zero-extended. This breaks signed comparisons so truncate
3030 // back to the underlying type before doing the compare.
3031 if (CondLHS.getValueType() != MemVT) {
3032 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
3033 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
3034 }
3035 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
3036 }
3037 } else {
3038 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
3039
3040 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
3041 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
3042
3043 SDValue CmpOp = getValue(CB.CmpMHS);
3044 EVT VT = CmpOp.getValueType();
3045
3046 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
3047 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
3048 ISD::SETLE);
3049 } else {
3050 SDValue SUB = DAG.getNode(ISD::SUB, dl,
3051 VT, CmpOp, DAG.getConstant(Low, dl, VT));
3052 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
3053 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
3054 }
3055 }
3056
3057 // Update successor info
3058 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
3059 // TrueBB and FalseBB are always different unless the incoming IR is
3060 // degenerate. This only happens when running llc on weird IR.
3061 if (CB.TrueBB != CB.FalseBB)
3062 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
3063 SwitchBB->normalizeSuccProbs();
3064
3065 // If the lhs block is the next block, invert the condition so that we can
3066 // fall through to the lhs instead of the rhs block.
3067 if (CB.TrueBB == NextBlock(SwitchBB)) {
3068 std::swap(CB.TrueBB, CB.FalseBB);
3069 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
3070 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
3071 }
3072
3073 SDNodeFlags Flags;
3075 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
3076 Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
3077
3078 setValue(CurInst, BrCond);
3079
3080 // Insert the false branch. Do this even if it's a fall through branch,
3081 // this makes it easier to do DAG optimizations which require inverting
3082 // the branch condition.
3083 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3084 DAG.getBasicBlock(CB.FalseBB));
3085
3086 DAG.setRoot(BrCond);
3087}
3088
3089/// visitJumpTable - Emit JumpTable node in the current MBB
3091 // Emit the code for the jump table
3092 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3093 assert(JT.Reg && "Should lower JT Header first!");
3094 EVT PTy = DAG.getTargetLoweringInfo().getJumpTableRegTy(DAG.getDataLayout());
3095 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
3096 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
3097 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
3098 Index.getValue(1), Table, Index);
3099 DAG.setRoot(BrJumpTable);
3100}
3101
3102/// visitJumpTableHeader - This function emits necessary code to produce index
3103/// in the JumpTable from switch case.
3105 JumpTableHeader &JTH,
3106 MachineBasicBlock *SwitchBB) {
3107 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3108 const SDLoc &dl = *JT.SL;
3109
3110 // Subtract the lowest switch case value from the value being switched on.
3111 SDValue SwitchOp = getValue(JTH.SValue);
3112 EVT VT = SwitchOp.getValueType();
3113 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
3114 DAG.getConstant(JTH.First, dl, VT));
3115
3116 // The SDNode we just created, which holds the value being switched on minus
3117 // the smallest case value, needs to be copied to a virtual register so it
3118 // can be used as an index into the jump table in a subsequent basic block.
3119 // This value may be smaller or larger than the target's pointer type, and
3120 // therefore require extension or truncating.
3121 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3122 SwitchOp =
3123 DAG.getZExtOrTrunc(Sub, dl, TLI.getJumpTableRegTy(DAG.getDataLayout()));
3124
3125 Register JumpTableReg =
3126 FuncInfo.CreateReg(TLI.getJumpTableRegTy(DAG.getDataLayout()));
3127 SDValue CopyTo =
3128 DAG.getCopyToReg(getControlRoot(), dl, JumpTableReg, SwitchOp);
3129 JT.Reg = JumpTableReg;
3130
3131 if (!JTH.FallthroughUnreachable) {
3132 // Emit the range check for the jump table, and branch to the default block
3133 // for the switch statement if the value being switched on exceeds the
3134 // largest case in the switch.
3135 SDValue CMP = DAG.getSetCC(
3136 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3137 Sub.getValueType()),
3138 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3139
3140 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3141 MVT::Other, CopyTo, CMP,
3142 DAG.getBasicBlock(JT.Default));
3143
3144 // Avoid emitting unnecessary branches to the next block.
3145 if (JT.MBB != NextBlock(SwitchBB))
3146 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3147 DAG.getBasicBlock(JT.MBB));
3148
3149 DAG.setRoot(BrCond);
3150 } else {
3151 // Avoid emitting unnecessary branches to the next block.
3152 if (JT.MBB != NextBlock(SwitchBB))
3153 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3154 DAG.getBasicBlock(JT.MBB)));
3155 else
3156 DAG.setRoot(CopyTo);
3157 }
3158}
3159
3160/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3161/// variable if there exists one.
3163 SDValue &Chain) {
3165 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3166 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3168 Value *Global =
3169 TLI.getSDagStackGuard(*MF.getFunction().getParent(), DAG.getLibcalls());
3171 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3172 if (Global) {
3173 MachinePointerInfo MPInfo(Global);
3177 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
3178 DAG.setNodeMemRefs(Node, {MemRef});
3179 }
3180 if (PtrTy != PtrMemTy)
3181 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3182 return SDValue(Node, 0);
3183}
3184
3185/// Codegen a new tail for a stack protector check ParentMBB which has had its
3186/// tail spliced into a stack protector check success bb.
3187///
3188/// For a high level explanation of how this fits into the stack protector
3189/// generation see the comment on the declaration of class
3190/// StackProtectorDescriptor.
3192 MachineBasicBlock *ParentBB) {
3193
3194 // First create the loads to the guard/stack slot for the comparison.
3195 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3196 auto &DL = DAG.getDataLayout();
3197 EVT PtrTy = TLI.getFrameIndexTy(DL);
3198 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
3199
3200 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3201 int FI = MFI.getStackProtectorIndex();
3202
3203 SDValue Guard;
3204 SDLoc dl = getCurSDLoc();
3205 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3206 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3207 Align Align = DL.getPrefTypeAlign(
3208 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
3209
3210 // Generate code to load the content of the guard slot.
3211 SDValue GuardVal = DAG.getLoad(
3212 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3213 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
3215
3216 // If cookie mixing is enabled, unmix the stored GuardVal to get back the
3217 // original cookie for comparison. The prologue stored (FP - Cookie) or
3218 // (FP XOR Cookie), so we apply the same operation again to unmix:
3219 // FP - (FP - Cookie) = Cookie, or (FP XOR Cookie) XOR FP = Cookie.
3220 if (TLI.useStackGuardMixFP())
3221 GuardVal = TLI.emitStackGuardMixFP(DAG, GuardVal, dl);
3222
3223 // If we're using function-based instrumentation, call the guard check
3224 // function
3226 // Get the guard check function from the target and verify it exists since
3227 // we're using function-based instrumentation
3228 const Function *GuardCheckFn =
3229 TLI.getSSPStackGuardCheck(M, DAG.getLibcalls());
3230 assert(GuardCheckFn && "Guard check function is null");
3231
3232 // The target provides a guard check function to validate the guard value.
3233 // Generate a call to that function with the content of the guard slot as
3234 // argument.
3235 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3236 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3237
3239 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
3240 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3241 Entry.IsInReg = true;
3242 Args.push_back(Entry);
3243
3246 .setChain(DAG.getEntryNode())
3247 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3248 getValue(GuardCheckFn), std::move(Args));
3249
3250 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3251 DAG.setRoot(Result.second);
3252 return;
3253 }
3254
3255 // Load the fresh guard value for comparison.
3256 // For targets that mix the cookie in LOAD_STACK_GUARD expansion, we need to
3257 // load directly without using LOAD_STACK_GUARD to avoid unwanted mixing.
3258 SDValue Chain = DAG.getEntryNode();
3259 if (TLI.useStackGuardMixFP()) {
3260 // Mixing targets: load cookie directly to avoid mixing in LOAD_STACK_GUARD
3261 if (const Value *IRGuard = TLI.getSDagStackGuard(M, DAG.getLibcalls())) {
3262 SDValue GuardPtr = getValue(IRGuard);
3263 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3264 MachinePointerInfo(IRGuard, 0), Align,
3266 } else {
3267 LLVMContext &Ctx = *DAG.getContext();
3268 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
3269 Guard = DAG.getPOISON(PtrMemTy);
3270 }
3271 } else {
3272 // Non-mixing targets: use LOAD_STACK_GUARD or direct load as usual
3273 if (TLI.useLoadStackGuardNode(M)) {
3274 Guard = getLoadStackGuard(DAG, dl, Chain);
3275 } else {
3276 if (const Value *IRGuard = TLI.getSDagStackGuard(M, DAG.getLibcalls())) {
3277 SDValue GuardPtr = getValue(IRGuard);
3278 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3279 MachinePointerInfo(IRGuard, 0), Align,
3281 } else {
3282 LLVMContext &Ctx = *DAG.getContext();
3283 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
3284 Guard = DAG.getPOISON(PtrMemTy);
3285 }
3286 }
3287 }
3288
3289 // Now both Guard (fresh cookie) and GuardVal (unmixed from stored value)
3290 // contain unmixed cookie values that can be compared directly.
3291
3292 // Perform the comparison via a getsetcc.
3293 SDValue Cmp = DAG.getSetCC(
3294 dl, TLI.getSetCCResultType(DL, *DAG.getContext(), Guard.getValueType()),
3295 Guard, GuardVal, ISD::SETNE);
3296
3297 // If the guard/stackslot do not equal, branch to failure MBB.
3298 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
3299 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3300 // Otherwise branch to success MBB.
3301 SDValue Br = DAG.getNode(ISD::BR, dl,
3302 MVT::Other, BrCond,
3303 DAG.getBasicBlock(SPD.getSuccessMBB()));
3304
3305 DAG.setRoot(Br);
3306}
3307
3308/// Codegen the failure basic block for a stack protector check.
3309///
3310/// A failure stack protector machine basic block consists simply of a call to
3311/// __stack_chk_fail().
3312///
3313/// For a high level explanation of how this fits into the stack protector
3314/// generation see the comment on the declaration of class
3315/// StackProtectorDescriptor.
3318
3319 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3320 MachineBasicBlock *ParentBB = SPD.getParentMBB();
3321 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3322 SDValue Chain;
3323
3324 // For -Oz builds with a guard check function, we use function-based
3325 // instrumentation. Otherwise, if we have a guard check function, we call it
3326 // in the failure block.
3327 auto *GuardCheckFn = TLI.getSSPStackGuardCheck(M, DAG.getLibcalls());
3328 if (GuardCheckFn && !SPD.shouldEmitFunctionBasedCheckStackProtector()) {
3329 // First create the loads to the guard/stack slot for the comparison.
3330 auto &DL = DAG.getDataLayout();
3331 EVT PtrTy = TLI.getFrameIndexTy(DL);
3332 EVT PtrMemTy = TLI.getPointerMemTy(DL, DL.getAllocaAddrSpace());
3333
3334 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3335 int FI = MFI.getStackProtectorIndex();
3336
3337 SDLoc dl = getCurSDLoc();
3338 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3339 Align Align = DL.getPrefTypeAlign(
3340 PointerType::get(M.getContext(), DL.getAllocaAddrSpace()));
3341
3342 // Generate code to load the content of the guard slot.
3343 SDValue GuardVal = DAG.getLoad(
3344 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3345 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
3347
3348 if (TLI.useStackGuardMixFP())
3349 GuardVal = TLI.emitStackGuardMixFP(DAG, GuardVal, dl);
3350
3351 // The target provides a guard check function to validate the guard value.
3352 // Generate a call to that function with the content of the guard slot as
3353 // argument.
3354 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3355 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3356
3358 TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
3359 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3360 Entry.IsInReg = true;
3361 Args.push_back(Entry);
3362
3365 .setChain(DAG.getEntryNode())
3366 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3367 getValue(GuardCheckFn), std::move(Args));
3368
3369 Chain = TLI.LowerCallTo(CLI).second;
3370 } else {
3372 CallOptions.setDiscardResult(true);
3373 Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3374 {}, CallOptions, getCurSDLoc())
3375 .second;
3376 }
3377
3378 // Emit a trap instruction if we are required to do so.
3379 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3380 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3381 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3382
3383 DAG.setRoot(Chain);
3384}
3385
3386/// visitBitTestHeader - This function emits necessary code to produce value
3387/// suitable for "bit tests"
3389 MachineBasicBlock *SwitchBB) {
3390 SDLoc dl = getCurSDLoc();
3391
3392 // Subtract the minimum value.
3393 SDValue SwitchOp = getValue(B.SValue);
3394 EVT VT = SwitchOp.getValueType();
3395 SDValue RangeSub =
3396 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3397
3398 // Determine the type of the test operands.
3399 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3400 bool UsePtrType = false;
3401 if (!TLI.isTypeLegal(VT)) {
3402 UsePtrType = true;
3403 } else {
3404 for (const BitTestCase &Case : B.Cases)
3405 if (!isUIntN(VT.getSizeInBits(), Case.Mask)) {
3406 // Switch table case range are encoded into series of masks.
3407 // Just use pointer type, it's guaranteed to fit.
3408 UsePtrType = true;
3409 break;
3410 }
3411 }
3412 SDValue Sub = RangeSub;
3413 if (UsePtrType) {
3414 VT = TLI.getPointerTy(DAG.getDataLayout());
3415 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3416 }
3417
3418 B.RegVT = VT.getSimpleVT();
3419 B.Reg = FuncInfo.CreateReg(B.RegVT);
3420 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3421
3422 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3423
3424 if (!B.FallthroughUnreachable)
3425 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3426 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3427 SwitchBB->normalizeSuccProbs();
3428
3429 SDValue Root = CopyTo;
3430 if (!B.FallthroughUnreachable) {
3431 // Conditional branch to the default block.
3432 SDValue RangeCmp = DAG.getSetCC(dl,
3433 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
3434 RangeSub.getValueType()),
3435 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3436 ISD::SETUGT);
3437
3438 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3439 DAG.getBasicBlock(B.Default));
3440 }
3441
3442 // Avoid emitting unnecessary branches to the next block.
3443 if (MBB != NextBlock(SwitchBB))
3444 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3445
3446 DAG.setRoot(Root);
3447}
3448
3449/// visitBitTestCase - this function produces one "bit test"
3451 MachineBasicBlock *NextMBB,
3452 BranchProbability BranchProbToNext,
3453 Register Reg, BitTestCase &B,
3454 MachineBasicBlock *SwitchBB) {
3455 SDLoc dl = getCurSDLoc();
3456 MVT VT = BB.RegVT;
3457 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3458 SDValue Cmp;
3459 unsigned PopCount = llvm::popcount(B.Mask);
3460 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3461 if (PopCount == 1) {
3462 // Testing for a single bit; just compare the shift count with what it
3463 // would need to be to shift a 1 bit in that position.
3464 Cmp = DAG.getSetCC(
3465 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3466 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3467 ISD::SETEQ);
3468 } else if (PopCount == BB.Range) {
3469 // There is only one zero bit in the range, test for it directly.
3470 Cmp = DAG.getSetCC(
3471 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3472 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3473 } else {
3474 // Make desired shift
3475 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3476 DAG.getConstant(1, dl, VT), ShiftOp);
3477
3478 // Emit bit tests and jumps
3479 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3480 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3481 Cmp = DAG.getSetCC(
3482 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
3483 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3484 }
3485
3486 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3487 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3488 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3489 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3490 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3491 // one as they are relative probabilities (and thus work more like weights),
3492 // and hence we need to normalize them to let the sum of them become one.
3493 SwitchBB->normalizeSuccProbs();
3494
3495 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3496 MVT::Other, getControlRoot(),
3497 Cmp, DAG.getBasicBlock(B.TargetBB));
3498
3499 // Avoid emitting unnecessary branches to the next block.
3500 if (NextMBB != NextBlock(SwitchBB))
3501 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3502 DAG.getBasicBlock(NextMBB));
3503
3504 DAG.setRoot(BrAnd);
3505}
3506
3507void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3508 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3509
3510 // Retrieve successors. Look through artificial IR level blocks like
3511 // catchswitch for successors.
3512 MachineBasicBlock *Return = FuncInfo.getMBB(I.getSuccessor(0));
3513 const BasicBlock *EHPadBB = I.getSuccessor(1);
3514 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(EHPadBB);
3515
3516 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3517 // have to do anything here to lower funclet bundles.
3518 failForInvalidBundles(I, "invokes",
3524
3525 const Value *Callee(I.getCalledOperand());
3526 const Function *Fn = dyn_cast<Function>(Callee);
3527 if (isa<InlineAsm>(Callee))
3528 visitInlineAsm(I, EHPadBB);
3529 else if (Fn && Fn->isIntrinsic()) {
3530 switch (Fn->getIntrinsicID()) {
3531 default:
3532 llvm_unreachable("Cannot invoke this intrinsic");
3533 case Intrinsic::donothing:
3534 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3535 case Intrinsic::seh_try_begin:
3536 case Intrinsic::seh_scope_begin:
3537 case Intrinsic::seh_try_end:
3538 case Intrinsic::seh_scope_end:
3539 if (EHPadMBB)
3540 // a block referenced by EH table
3541 // so dtor-funclet not removed by opts
3542 EHPadMBB->setMachineBlockAddressTaken();
3543 break;
3544 case Intrinsic::experimental_patchpoint_void:
3545 case Intrinsic::experimental_patchpoint:
3546 visitPatchpoint(I, EHPadBB);
3547 break;
3548 case Intrinsic::experimental_gc_statepoint:
3550 break;
3551 // wasm_throw, wasm_rethrow: This is usually done in visitTargetIntrinsic,
3552 // but these intrinsics are special because they can be invoked, so we
3553 // manually lower it to a DAG node here.
3554 case Intrinsic::wasm_throw: {
3556 std::array<SDValue, 4> Ops = {
3557 getControlRoot(), // inchain for the terminator node
3558 DAG.getTargetConstant(Intrinsic::wasm_throw, getCurSDLoc(),
3559 TLI.getPointerTy(DAG.getDataLayout())),
3560 getValue(I.getArgOperand(0)), // tag
3561 getValue(I.getArgOperand(1)) // thrown value
3562 };
3563 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3564 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3565 break;
3566 }
3567 case Intrinsic::wasm_rethrow: {
3568 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3569 std::array<SDValue, 2> Ops = {
3570 getControlRoot(), // inchain for the terminator node
3571 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3572 TLI.getPointerTy(DAG.getDataLayout()))};
3573 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3574 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
3575 break;
3576 }
3577 // EH labels mark it as an invoke site, like a call.
3578 case Intrinsic::bpf_unwind_exit: {
3579 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3580 MCSymbol *BeginLabel;
3581 std::array<SDValue, 2> Ops = {
3582 lowerStartEH(getControlRoot(), EHPadBB, BeginLabel),
3583 DAG.getTargetConstant(Intrinsic::bpf_unwind_exit, getCurSDLoc(),
3584 TLI.getPointerTy(DAG.getDataLayout()))};
3585 SDValue Chain =
3586 DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), MVT::Other, Ops);
3587 DAG.setRoot(lowerEndEH(Chain, &I, EHPadBB, BeginLabel));
3588 break;
3589 }
3590 }
3591 } else if (I.hasDeoptState()) {
3592 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3593 // Eventually we will support lowering the @llvm.experimental.deoptimize
3594 // intrinsic, and right now there are no plans to support other intrinsics
3595 // with deopt state.
3596 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3597 } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
3599 } else {
3600 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3601 }
3602
3603 // If the value of the invoke is used outside of its defining block, make it
3604 // available as a virtual register.
3605 // We already took care of the exported value for the statepoint instruction
3606 // during call to the LowerStatepoint.
3607 if (!isa<GCStatepointInst>(I)) {
3609 }
3610
3612 BranchProbabilityInfo *BPI = FuncInfo.BPI;
3613 BranchProbability EHPadBBProb =
3614 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3616 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3617
3618 // Update successor info.
3619 addSuccessorWithProb(InvokeMBB, Return);
3620 for (auto &UnwindDest : UnwindDests) {
3621 UnwindDest.first->setIsEHPad();
3622 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3623 }
3624 InvokeMBB->normalizeSuccProbs();
3625
3626 // Drop into normal successor.
3627 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
3628 DAG.getBasicBlock(Return)));
3629}
3630
3631/// The intrinsics currently supported by callbr are implicit control flow
3632/// intrinsics such as amdgcn.kill.
3633/// - they should be called (no "dontcall-" attributes)
3634/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic())
3635/// - they do not need custom argument handling (no
3636/// TLI.CollectTargetIntrinsicOperands())
3637void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) {
3638#ifndef NDEBUG
3640 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3641 Infos, I, DAG.getMachineFunction(), I.getIntrinsicID());
3642 assert(Infos.empty() && "Intrinsic touches memory");
3643#endif
3644
3645 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
3646
3648 getTargetIntrinsicOperands(I, HasChain, OnlyLoad);
3649 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
3650
3651 // Create the node.
3652 SDValue Result =
3653 getTargetNonMemIntrinsicNode(*I.getType(), HasChain, Ops, VTs);
3654 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
3655
3656 setValue(&I, Result);
3657}
3658
3659void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3660 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3661
3662 if (I.isInlineAsm()) {
3663 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3664 // have to do anything here to lower funclet bundles.
3665 failForInvalidBundles(I, "callbrs",
3667 visitInlineAsm(I);
3668 } else {
3669 assert(!I.hasOperandBundles() &&
3670 "Can't have operand bundles for intrinsics");
3671 visitCallBrIntrinsic(I);
3672 }
3674
3675 // Retrieve successors.
3676 SmallPtrSet<BasicBlock *, 8> Dests;
3677 Dests.insert(I.getDefaultDest());
3678 MachineBasicBlock *Return = FuncInfo.getMBB(I.getDefaultDest());
3679
3680 // Update successor info.
3681 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3682 // TODO: For most of the cases where there is an intrinsic callbr, we're
3683 // having exactly one indirect target, which will be unreachable. As soon as
3684 // this changes, we might need to enhance
3685 // Target->setIsInlineAsmBrIndirectTarget or add something similar for
3686 // intrinsic indirect branches.
3687 if (I.isInlineAsm()) {
3688 for (BasicBlock *Dest : I.getIndirectDests()) {
3689 MachineBasicBlock *Target = FuncInfo.getMBB(Dest);
3690 Target->setIsInlineAsmBrIndirectTarget();
3691 // If we introduce a type of asm goto statement that is permitted to use
3692 // an indirect call instruction to jump to its labels, then we should add
3693 // a call to Target->setMachineBlockAddressTaken() here, to mark the
3694 // target block as requiring a BTI.
3695
3696 Target->setLabelMustBeEmitted();
3697 // Don't add duplicate machine successors.
3698 if (Dests.insert(Dest).second)
3699 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3700 }
3701 }
3702 CallBrMBB->normalizeSuccProbs();
3703
3704 // Drop into default successor.
3705 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
3706 MVT::Other, getControlRoot(),
3707 DAG.getBasicBlock(Return)));
3708}
3709
3710void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3711 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3712}
3713
3714void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3715 assert(FuncInfo.MBB->isEHPad() &&
3716 "Call to landingpad not in landing pad!");
3717
3718 // If there aren't registers to copy the values into (e.g., during SjLj
3719 // exceptions), then don't bother to create these DAG nodes.
3720 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3721 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3722 if (TLI.getExceptionPointerRegister(FuncInfo.ExceptionModel, PersonalityFn) ==
3723 0 &&
3725 PersonalityFn) == 0)
3726 return;
3727
3728 // If landingpad's return type is token type, we don't create DAG nodes
3729 // for its exception pointer and selector value. The extraction of exception
3730 // pointer or selector value from token type landingpads is not currently
3731 // supported.
3732 if (LP.getType()->isTokenTy())
3733 return;
3734
3735 // LangRef leaves the result type target-specific, so diagnose types this
3736 // lowering cannot represent instead of asserting.
3737 SDLoc dl = getCurSDLoc();
3739 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
3740 *LP.getFunction(),
3741 "landingpad result type must be a struct of an exception pointer and "
3742 "an integer selector",
3743 dl.getDebugLoc()));
3744 setValueToPoison(&LP, dl);
3745 return;
3746 }
3747
3748 SmallVector<EVT, 2> ValueVTs;
3749 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3750 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3751
3752 // Get the two live-in registers as SDValues. The physregs have already been
3753 // copied into virtual registers.
3754 SDValue Ops[2];
3755 if (FuncInfo.ExceptionPointerVirtReg) {
3756 Ops[0] = DAG.getZExtOrTrunc(
3757 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3758 FuncInfo.ExceptionPointerVirtReg,
3759 TLI.getPointerTy(DAG.getDataLayout())),
3760 dl, ValueVTs[0]);
3761 } else {
3762 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3763 }
3764 Ops[1] = DAG.getZExtOrTrunc(
3765 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
3766 FuncInfo.ExceptionSelectorVirtReg,
3767 TLI.getPointerTy(DAG.getDataLayout())),
3768 dl, ValueVTs[1]);
3769
3770 // Merge into one.
3771 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
3772 DAG.getVTList(ValueVTs), Ops);
3773 setValue(&LP, Res);
3774}
3775
3778 // Update JTCases.
3779 for (JumpTableBlock &JTB : SL->JTCases)
3780 if (JTB.first.HeaderBB == First)
3781 JTB.first.HeaderBB = Last;
3782
3783 // Update BitTestCases.
3784 for (BitTestBlock &BTB : SL->BitTestCases)
3785 if (BTB.Parent == First)
3786 BTB.Parent = Last;
3787}
3788
3789void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3790 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3791
3792 // Update machine-CFG edges with unique successors.
3794 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3795 BasicBlock *BB = I.getSuccessor(i);
3796 bool Inserted = Done.insert(BB).second;
3797 if (!Inserted)
3798 continue;
3799
3800 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3801 addSuccessorWithProb(IndirectBrMBB, Succ);
3802 }
3803 IndirectBrMBB->normalizeSuccProbs();
3804
3806 MVT::Other, getControlRoot(),
3807 getValue(I.getAddress())));
3808}
3809
3810void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3811 if (!I.shouldLowerToTrap(DAG.getTarget().Options.TrapUnreachable,
3812 DAG.getTarget().Options.NoTrapAfterNoreturn))
3813 return;
3814
3815 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3816}
3817
3818void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3819 SDNodeFlags Flags;
3820 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3821 Flags.copyFMF(*FPOp);
3822
3823 SDValue Op = getValue(I.getOperand(0));
3824 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3825 Op, Flags);
3826 setValue(&I, UnNodeValue);
3827}
3828
3829void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3830 SDNodeFlags Flags;
3831 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3832 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3833 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3834 }
3835 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3836 Flags.setExact(ExactOp->isExact());
3837 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3838 Flags.setDisjoint(DisjointOp->isDisjoint());
3839 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3840 Flags.copyFMF(*FPOp);
3841
3842 SDValue Op1 = getValue(I.getOperand(0));
3843 SDValue Op2 = getValue(I.getOperand(1));
3844 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3845 Op1, Op2, Flags);
3846 setValue(&I, BinNodeValue);
3847}
3848
3849void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3850 SDValue Op1 = getValue(I.getOperand(0));
3851 SDValue Op2 = getValue(I.getOperand(1));
3852
3853 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3854 Op1.getValueType(), DAG.getDataLayout());
3855
3856 // Coerce the shift amount to the right type if we can. This exposes the
3857 // truncate or zext to optimization early.
3858 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3860 "Unexpected shift type");
3861 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3862 }
3863
3864 bool nuw = false;
3865 bool nsw = false;
3866 bool exact = false;
3867
3868 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3869
3870 if (const OverflowingBinaryOperator *OFBinOp =
3872 nuw = OFBinOp->hasNoUnsignedWrap();
3873 nsw = OFBinOp->hasNoSignedWrap();
3874 }
3875 if (const PossiblyExactOperator *ExactOp =
3877 exact = ExactOp->isExact();
3878 }
3879 SDNodeFlags Flags;
3880 Flags.setExact(exact);
3881 Flags.setNoSignedWrap(nsw);
3882 Flags.setNoUnsignedWrap(nuw);
3883 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3884 Flags);
3885 setValue(&I, Res);
3886}
3887
3888void SelectionDAGBuilder::visitSDiv(const User &I) {
3889 SDValue Op1 = getValue(I.getOperand(0));
3890 SDValue Op2 = getValue(I.getOperand(1));
3891
3892 SDNodeFlags Flags;
3893 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3894 cast<PossiblyExactOperator>(&I)->isExact());
3895 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3896 Op2, Flags));
3897}
3898
3899void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3900 ICmpInst::Predicate predicate = I.getPredicate();
3901 SDValue Op1 = getValue(I.getOperand(0));
3902 SDValue Op2 = getValue(I.getOperand(1));
3903 ISD::CondCode Opcode = getICmpCondCode(predicate);
3904
3905 auto &TLI = DAG.getTargetLoweringInfo();
3906 EVT MemVT =
3907 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3908
3909 // If a pointer's DAG type is larger than its memory type then the DAG values
3910 // are zero-extended. This breaks signed comparisons so truncate back to the
3911 // underlying type before doing the compare.
3912 if (Op1.getValueType() != MemVT) {
3913 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3914 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3915 }
3916
3917 SDNodeFlags Flags;
3918 Flags.setSameSign(I.hasSameSign());
3919
3920 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3921 I.getType());
3922 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode,
3923 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3924}
3925
3926void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3927 FCmpInst::Predicate predicate = I.getPredicate();
3928 SDValue Op1 = getValue(I.getOperand(0));
3929 SDValue Op2 = getValue(I.getOperand(1));
3930
3931 ISD::CondCode Condition = getFCmpCondCode(predicate);
3932 auto *FPMO = cast<FPMathOperator>(&I);
3933 if (FPMO->hasNoNaNs() ||
3934 (DAG.isKnownNeverNaN(Op1) && DAG.isKnownNeverNaN(Op2)))
3935 Condition = getFCmpCodeWithoutNaN(Condition);
3936
3937 SDNodeFlags Flags;
3938 Flags.copyFMF(*FPMO);
3939
3940 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3941 I.getType());
3942 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition,
3943 /*Chain=*/{}, /*IsSignaling=*/false, Flags));
3944}
3945
3946// Check if the condition of the select has one use or two users that are both
3947// selects with the same condition.
3948static bool hasOnlySelectUsers(const Value *Cond) {
3949 return llvm::all_of(Cond->users(), [](const Value *V) {
3950 return isa<SelectInst>(V);
3951 });
3952}
3953
3954void SelectionDAGBuilder::visitSelect(const User &I) {
3955 SmallVector<EVT, 4> ValueVTs;
3956 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3957 ValueVTs);
3958 unsigned NumValues = ValueVTs.size();
3959 if (NumValues == 0) return;
3960
3962 SDValue Cond = getValue(I.getOperand(0));
3963 SDValue LHSVal = getValue(I.getOperand(1));
3964 SDValue RHSVal = getValue(I.getOperand(2));
3965 SmallVector<SDValue, 1> BaseOps(1, Cond);
3967 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3968
3969 bool IsUnaryAbs = false;
3970 bool Negate = false;
3971
3972 SDNodeFlags Flags;
3973 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3974 Flags.copyFMF(*FPOp);
3975
3976 Flags.setUnpredictable(
3977 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3978
3979 // Min/max matching is only viable if all output VTs are the same.
3980 if (all_equal(ValueVTs)) {
3981 EVT VT = ValueVTs[0];
3982 LLVMContext &Ctx = *DAG.getContext();
3983 auto &TLI = DAG.getTargetLoweringInfo();
3984
3985 // We care about the legality of the operation after it has been type
3986 // legalized.
3988 VT = TLI.getTypeToTransformTo(Ctx, VT);
3989
3990 // If the vselect is legal, assume we want to leave this as a vector setcc +
3991 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3992 // min/max is legal on the scalar type.
3993 bool UseScalarMinMax = VT.isVector() &&
3995
3996 // ValueTracking's select pattern matching does not account for -0.0,
3997 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3998 // -0.0 is less than +0.0.
3999 const Value *LHS, *RHS;
4000 auto SPR = matchSelectPattern(&I, LHS, RHS);
4002 switch (SPR.Flavor) {
4003 case SPF_UMAX: Opc = ISD::UMAX; break;
4004 case SPF_UMIN: Opc = ISD::UMIN; break;
4005 case SPF_SMAX: Opc = ISD::SMAX; break;
4006 case SPF_SMIN: Opc = ISD::SMIN; break;
4007 case SPF_FMINNUM:
4009 break;
4010
4011 switch (SPR.NaNBehavior) {
4012 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4013 case SPNB_RETURNS_ANY:
4014 case SPNB_RETURNS_NAN:
4015 break;
4016 case SPNB_RETURNS_OTHER:
4018 Flags.setNoSignedZeros(true);
4019 break;
4020 }
4021 break;
4022 case SPF_FMAXNUM:
4024 break;
4025
4026 switch (SPR.NaNBehavior) {
4027 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
4028 case SPNB_RETURNS_NAN:
4029 case SPNB_RETURNS_ANY:
4030 break;
4031 case SPNB_RETURNS_OTHER:
4033 Flags.setNoSignedZeros(true);
4034 break;
4035 }
4036 break;
4037 case SPF_NABS:
4038 Negate = true;
4039 [[fallthrough]];
4040 case SPF_ABS:
4041 IsUnaryAbs = true;
4042 Opc = ISD::ABS;
4043 break;
4044 default: break;
4045 }
4046
4047 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
4049 (UseScalarMinMax &&
4051 // If the underlying comparison instruction is used by any other
4052 // instruction, the consumed instructions won't be destroyed, so it is
4053 // not profitable to convert to a min/max.
4055 OpCode = Opc;
4056 LHSVal = getValue(LHS);
4057 RHSVal = getValue(RHS);
4058 BaseOps.clear();
4059 }
4060
4061 if (IsUnaryAbs) {
4062 OpCode = Opc;
4063 LHSVal = getValue(LHS);
4064 BaseOps.clear();
4065 }
4066 }
4067
4068 if (IsUnaryAbs) {
4069 for (unsigned i = 0; i != NumValues; ++i) {
4070 SDLoc dl = getCurSDLoc();
4071 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
4072 Values[i] =
4073 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
4074 if (Negate)
4075 Values[i] = DAG.getNegative(Values[i], dl, VT);
4076 }
4077 } else {
4078 for (unsigned i = 0; i != NumValues; ++i) {
4079 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
4080 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
4081 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
4082 Values[i] = DAG.getNode(
4083 OpCode, getCurSDLoc(),
4084 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
4085 }
4086 }
4087
4089 DAG.getVTList(ValueVTs), Values));
4090}
4091
4092void SelectionDAGBuilder::visitTrunc(const User &I) {
4093 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
4094 SDValue N = getValue(I.getOperand(0));
4095 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4096 I.getType());
4097 SDNodeFlags Flags;
4098 if (auto *Trunc = dyn_cast<TruncInst>(&I)) {
4099 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
4100 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
4101 }
4102
4103 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N, Flags));
4104}
4105
4106void SelectionDAGBuilder::visitZExt(const User &I) {
4107 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4108 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
4109 SDValue N = getValue(I.getOperand(0));
4110 auto &TLI = DAG.getTargetLoweringInfo();
4111 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4112
4113 SDNodeFlags Flags;
4114 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
4115 Flags.setNonNeg(PNI->hasNonNeg());
4116
4117 // Eagerly use nonneg information to canonicalize towards sign_extend if
4118 // that is the target's preference.
4119 // TODO: Let the target do this later.
4120 if (Flags.hasNonNeg() &&
4121 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
4122 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
4123 return;
4124 }
4125
4126 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
4127}
4128
4129void SelectionDAGBuilder::visitSExt(const User &I) {
4130 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
4131 // SExt also can't be a cast to bool for same reason. So, nothing much to do
4132 SDValue N = getValue(I.getOperand(0));
4133 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4134 I.getType());
4135 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
4136}
4137
4138void SelectionDAGBuilder::visitFPTrunc(const User &I) {
4139 // FPTrunc is never a no-op cast, no need to check
4140 SDValue N = getValue(I.getOperand(0));
4141 SDLoc dl = getCurSDLoc();
4142 SDNodeFlags Flags;
4143 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
4144 Flags.copyFMF(*FPOp);
4145 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4146 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4147 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
4148 DAG.getTargetConstant(
4149 0, dl, TLI.getPointerTy(DAG.getDataLayout())),
4150 Flags));
4151}
4152
4153void SelectionDAGBuilder::visitFPExt(const User &I) {
4154 // FPExt is never a no-op cast, no need to check
4155 SDValue N = getValue(I.getOperand(0));
4156 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4157 I.getType());
4158 SDNodeFlags Flags;
4159 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
4160 Flags.copyFMF(*FPOp);
4161 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N, Flags));
4162}
4163
4164void SelectionDAGBuilder::visitFPToUI(const User &I) {
4165 // FPToUI is never a no-op cast, no need to check
4166 SDValue N = getValue(I.getOperand(0));
4167 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4168 I.getType());
4169 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
4170}
4171
4172void SelectionDAGBuilder::visitFPToSI(const User &I) {
4173 // FPToSI is never a no-op cast, no need to check
4174 SDValue N = getValue(I.getOperand(0));
4175 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4176 I.getType());
4177 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
4178}
4179
4180void SelectionDAGBuilder::visitUIToFP(const User &I) {
4181 // UIToFP is never a no-op cast, no need to check
4182 SDValue N = getValue(I.getOperand(0));
4183 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4184 I.getType());
4185 SDNodeFlags Flags;
4186 Flags.setNonNeg(cast<PossiblyNonNegInst>(&I)->hasNonNeg());
4187 Flags.copyFMF(*cast<FPMathOperator>(&I));
4188
4189 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
4190}
4191
4192void SelectionDAGBuilder::visitSIToFP(const User &I) {
4193 // SIToFP is never a no-op cast, no need to check
4194 SDValue N = getValue(I.getOperand(0));
4195 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4196 I.getType());
4197 SDNodeFlags Flags;
4198 Flags.copyFMF(*cast<FPMathOperator>(&I));
4199
4200 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
4201}
4202
4203void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
4204 SDValue N = getValue(I.getOperand(0));
4205 // By definition the type of the ptrtoaddr must be equal to the address type.
4206 const auto &TLI = DAG.getTargetLoweringInfo();
4207 EVT AddrVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4208 // The address width must be smaller or equal to the pointer representation
4209 // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op).
4210 N = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), AddrVT, N);
4211 setValue(&I, N);
4212}
4213
4214void SelectionDAGBuilder::visitPtrToInt(const User &I) {
4215 // What to do depends on the size of the integer and the size of the pointer.
4216 // We can either truncate, zero extend, or no-op, accordingly.
4217 SDValue N = getValue(I.getOperand(0));
4218 auto &TLI = DAG.getTargetLoweringInfo();
4219 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4220 I.getType());
4221 EVT PtrMemVT =
4222 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
4223 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
4224 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
4225 setValue(&I, N);
4226}
4227
4228void SelectionDAGBuilder::visitIntToPtr(const User &I) {
4229 // What to do depends on the size of the integer and the size of the pointer.
4230 // We can either truncate, zero extend, or no-op, accordingly.
4231 SDValue N = getValue(I.getOperand(0));
4232 auto &TLI = DAG.getTargetLoweringInfo();
4233 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4234 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4235 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
4236 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
4237 setValue(&I, N);
4238}
4239
4240void SelectionDAGBuilder::visitBitCast(const User &I) {
4241 SDValue N = getValue(I.getOperand(0));
4242 SDLoc dl = getCurSDLoc();
4243 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
4244 I.getType());
4245
4246 // BitCast assures us that source and destination are the same size so this is
4247 // either a BITCAST or a no-op.
4248 if (DestVT != N.getValueType())
4249 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
4250 DestVT, N)); // convert types.
4251 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
4252 // might fold any kind of constant expression to an integer constant and that
4253 // is not what we are looking for. Only recognize a bitcast of a genuine
4254 // constant integer as an opaque constant.
4255 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
4256 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
4257 /*isOpaque*/true));
4258 else
4259 setValue(&I, N); // noop cast.
4260}
4261
4262void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
4263 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4264 const Value *SV = I.getOperand(0);
4265 SDValue N = getValue(SV);
4266 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4267
4268 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
4269 unsigned DestAS = I.getType()->getPointerAddressSpace();
4270
4271 if (!TM.isNoopAddrSpaceCast(DAG.getDataLayout(), SrcAS, DestAS)) {
4272 SDNodeFlags Flags;
4273 if (const auto *ASC = dyn_cast<AddrSpaceCastInst>(&I))
4274 Flags.setNonNull(ASC->hasNonNull());
4275 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS, Flags);
4276 }
4277
4278 setValue(&I, N);
4279}
4280
4281void SelectionDAGBuilder::visitInsertElement(const User &I) {
4282 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4283 SDValue InVec = getValue(I.getOperand(0));
4284 SDValue InVal = getValue(I.getOperand(1));
4285 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
4286 TLI.getVectorIdxTy(DAG.getDataLayout()));
4288 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4289 InVec, InVal, InIdx));
4290}
4291
4292void SelectionDAGBuilder::visitBitInsert(const User &I) {
4293 SDValue Base = getValue(I.getOperand(0));
4294 SDValue Val = getValue(I.getOperand(1));
4295 SDValue Offset = getValue(I.getOperand(2));
4296 EVT BaseVT = Base.getValueType();
4297 EVT ValVT = Val.getValueType();
4298 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4299 SDLoc dl = getCurSDLoc();
4300
4301 assert(BaseVT.getScalarSizeInBits() >= ValVT.getScalarSizeInBits() &&
4302 "bitinsert val wider than base should be rejected by verifier");
4303
4304 // If Val is a float, cast it to an integer of the same bitwidth
4305 // so DAG.getZExtOrTrunc can process it safely.
4306 if (!ValVT.isInteger()) {
4307 ValVT = ValVT.changeTypeToInteger();
4308 Val = DAG.getBitcast(ValVT, Val);
4309 }
4310
4311 // Legalize shift amount to the target's shift amount type.
4312 EVT ShiftAmtTy = TLI.getShiftAmountTy(BaseVT, DAG.getDataLayout());
4313 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Offset, dl, ShiftAmtTy);
4314
4315 unsigned BaseBitWidth = BaseVT.getScalarSizeInBits();
4316 unsigned ValBitWidth = ValVT.getScalarSizeInBits();
4317 APInt InsertMask = APInt::getLowBitsSet(BaseBitWidth, ValBitWidth);
4318 SDValue ShiftedMask =
4319 DAG.getNode(ISD::SHL, dl, BaseVT, DAG.getConstant(InsertMask, dl, BaseVT),
4320 LegalShiftAmount);
4321 SDValue ClearMask = DAG.getNOT(dl, ShiftedMask, BaseVT);
4322 SDValue ClearedBase = DAG.getNode(ISD::AND, dl, BaseVT, Base, ClearMask);
4323
4324 SDValue ExtVal = DAG.getZExtOrTrunc(Val, dl, BaseVT);
4325 SDValue ShiftedVal =
4326 DAG.getNode(ISD::SHL, dl, BaseVT, ExtVal, LegalShiftAmount);
4327 SDValue Result = DAG.getNode(ISD::OR, dl, BaseVT, ClearedBase, ShiftedVal);
4328 setValue(&I, Result);
4329}
4330
4331void SelectionDAGBuilder::visitBitExtract(const User &I) {
4332 SDValue Src = getValue(I.getOperand(0));
4333 SDValue Offset = getValue(I.getOperand(1));
4334 EVT SrcVT = Src.getValueType();
4335 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4336 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4337 SDLoc dl = getCurSDLoc();
4338
4339 assert(ResultVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() &&
4340 "bitextract result wider than source should be rejected by verifier");
4341
4342 // Legalize shift amount to the target's shift amount type.
4343 EVT ShiftAmtTy = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
4344 SDValue LegalShiftAmount = DAG.getZExtOrTrunc(Offset, dl, ShiftAmtTy);
4345
4346 // Shift right by Offset - brings target field to bit 0
4347 SDValue Shifted = DAG.getNode(ISD::SRL, dl, SrcVT, Src, LegalShiftAmount);
4348
4350 if (!ResultVT.isInteger()) {
4351 // Drop into the integer domain to safely truncate the shifted bits
4352 EVT IntResultVT = ResultVT.changeTypeToInteger();
4353 Result = DAG.getNode(ISD::TRUNCATE, dl, IntResultVT, Shifted);
4354 Result = DAG.getBitcast(ResultVT, Result);
4355 } else {
4356 // Normal integer path
4357 Result = DAG.getNode(ISD::TRUNCATE, dl, ResultVT, Shifted);
4358 }
4359
4360 setValue(&I, Result);
4361}
4362
4363void SelectionDAGBuilder::visitExtractElement(const User &I) {
4364 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4365 SDValue InVec = getValue(I.getOperand(0));
4366 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
4367 TLI.getVectorIdxTy(DAG.getDataLayout()));
4369 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4370 InVec, InIdx));
4371}
4372
4373void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4374 SDValue Src1 = getValue(I.getOperand(0));
4375 SDValue Src2 = getValue(I.getOperand(1));
4376 ArrayRef<int> Mask;
4377 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
4378 Mask = SVI->getShuffleMask();
4379 else
4380 Mask = cast<ConstantExpr>(I).getShuffleMask();
4381 SDLoc DL = getCurSDLoc();
4382 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4383 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4384 EVT SrcVT = Src1.getValueType();
4385
4386 if (all_of(Mask, equal_to(0)) && VT.isScalableVector()) {
4387 // Canonical splat form of first element of first input vector.
4388 SDValue FirstElt =
4389 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
4390 DAG.getVectorIdxConstant(0, DL));
4391 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4392 return;
4393 }
4394
4395 // For now, we only handle splats for scalable vectors.
4396 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4397 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4398 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4399
4400 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4401 unsigned MaskNumElts = Mask.size();
4402
4403 if (SrcNumElts == MaskNumElts) {
4404 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4405 return;
4406 }
4407
4408 // Normalize the shuffle vector since mask and vector length don't match.
4409 if (SrcNumElts < MaskNumElts) {
4410 // Mask is longer than the source vectors. We can use concatenate vector to
4411 // make the mask and vectors lengths match.
4412
4413 if (MaskNumElts % SrcNumElts == 0) {
4414 // Mask length is a multiple of the source vector length.
4415 // Check if the shuffle is some kind of concatenation of the input
4416 // vectors.
4417 unsigned NumConcat = MaskNumElts / SrcNumElts;
4418 bool IsConcat = true;
4419 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4420 for (unsigned i = 0; i != MaskNumElts; ++i) {
4421 int Idx = Mask[i];
4422 if (Idx < 0)
4423 continue;
4424 // Ensure the indices in each SrcVT sized piece are sequential and that
4425 // the same source is used for the whole piece.
4426 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4427 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4428 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4429 IsConcat = false;
4430 break;
4431 }
4432 // Remember which source this index came from.
4433 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4434 }
4435
4436 // The shuffle is concatenating multiple vectors together. Just emit
4437 // a CONCAT_VECTORS operation.
4438 if (IsConcat) {
4439 SmallVector<SDValue, 8> ConcatOps;
4440 for (auto Src : ConcatSrcs) {
4441 if (Src < 0)
4442 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4443 else if (Src == 0)
4444 ConcatOps.push_back(Src1);
4445 else
4446 ConcatOps.push_back(Src2);
4447 }
4448 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4449 return;
4450 }
4451 }
4452
4453 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4454 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4455 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4456 PaddedMaskNumElts);
4457
4458 // Pad both vectors with undefs to make them the same length as the mask.
4459 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4460
4461 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4462 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4463 MOps1[0] = Src1;
4464 MOps2[0] = Src2;
4465
4466 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4467 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4468
4469 // Readjust mask for new input vector length.
4470 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4471 for (unsigned i = 0; i != MaskNumElts; ++i) {
4472 int Idx = Mask[i];
4473 if (Idx >= (int)SrcNumElts)
4474 Idx -= SrcNumElts - PaddedMaskNumElts;
4475 MappedOps[i] = Idx;
4476 }
4477
4478 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4479
4480 // If the concatenated vector was padded, extract a subvector with the
4481 // correct number of elements.
4482 if (MaskNumElts != PaddedMaskNumElts)
4483 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
4484 DAG.getVectorIdxConstant(0, DL));
4485
4486 setValue(&I, Result);
4487 return;
4488 }
4489
4490 assert(SrcNumElts > MaskNumElts);
4491
4492 // Analyze the access pattern of the vector to see if we can extract
4493 // two subvectors and do the shuffle.
4494 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4495 bool CanExtract = true;
4496 for (int Idx : Mask) {
4497 unsigned Input = 0;
4498 if (Idx < 0)
4499 continue;
4500
4501 if (Idx >= (int)SrcNumElts) {
4502 Input = 1;
4503 Idx -= SrcNumElts;
4504 }
4505
4506 // If all the indices come from the same MaskNumElts sized portion of
4507 // the sources we can use extract. Also make sure the extract wouldn't
4508 // extract past the end of the source.
4509 int NewStartIdx = alignDown(Idx, MaskNumElts);
4510 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4511 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4512 CanExtract = false;
4513 // Make sure we always update StartIdx as we use it to track if all
4514 // elements are undef.
4515 StartIdx[Input] = NewStartIdx;
4516 }
4517
4518 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4519 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4520 return;
4521 }
4522 if (CanExtract) {
4523 // Extract appropriate subvector and generate a vector shuffle
4524 for (unsigned Input = 0; Input < 2; ++Input) {
4525 SDValue &Src = Input == 0 ? Src1 : Src2;
4526 if (StartIdx[Input] < 0)
4527 Src = DAG.getUNDEF(VT);
4528 else {
4529 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4530 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4531 }
4532 }
4533
4534 // Calculate new mask.
4535 SmallVector<int, 8> MappedOps(Mask);
4536 for (int &Idx : MappedOps) {
4537 if (Idx >= (int)SrcNumElts)
4538 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4539 else if (Idx >= 0)
4540 Idx -= StartIdx[0];
4541 }
4542
4543 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4544 return;
4545 }
4546
4547 // We can't use either concat vectors or extract subvectors so fall back to
4548 // replacing the shuffle with extract and build vector.
4549 // to insert and build vector.
4550 EVT EltVT = VT.getVectorElementType();
4552 for (int Idx : Mask) {
4553 SDValue Res;
4554
4555 if (Idx < 0) {
4556 Res = DAG.getUNDEF(EltVT);
4557 } else {
4558 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4559 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4560
4561 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4562 DAG.getVectorIdxConstant(Idx, DL));
4563 }
4564
4565 Ops.push_back(Res);
4566 }
4567
4568 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4569}
4570
4571void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4572 ArrayRef<unsigned> Indices = I.getIndices();
4573 const Value *Op0 = I.getOperand(0);
4574 const Value *Op1 = I.getOperand(1);
4575 Type *AggTy = I.getType();
4576 Type *ValTy = Op1->getType();
4577 bool IntoUndef = isa<UndefValue>(Op0);
4578 bool FromUndef = isa<UndefValue>(Op1);
4579
4580 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4581
4582 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4583 SmallVector<EVT, 4> AggValueVTs;
4584 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4585 SmallVector<EVT, 4> ValValueVTs;
4586 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4587
4588 unsigned NumAggValues = AggValueVTs.size();
4589 unsigned NumValValues = ValValueVTs.size();
4590 SmallVector<SDValue, 4> Values(NumAggValues);
4591
4592 // Ignore an insertvalue that produces an empty object
4593 if (!NumAggValues) {
4594 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4595 return;
4596 }
4597
4598 SDValue Agg = getValue(Op0);
4599 unsigned i = 0;
4600 // Copy the beginning value(s) from the original aggregate.
4601 for (; i != LinearIndex; ++i)
4602 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4603 SDValue(Agg.getNode(), Agg.getResNo() + i);
4604 // Copy values from the inserted value(s).
4605 if (NumValValues) {
4606 SDValue Val = getValue(Op1);
4607 for (; i != LinearIndex + NumValValues; ++i)
4608 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4609 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4610 }
4611 // Copy remaining value(s) from the original aggregate.
4612 for (; i != NumAggValues; ++i)
4613 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4614 SDValue(Agg.getNode(), Agg.getResNo() + i);
4615
4617 DAG.getVTList(AggValueVTs), Values));
4618}
4619
4620void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4621 ArrayRef<unsigned> Indices = I.getIndices();
4622 const Value *Op0 = I.getOperand(0);
4623 Type *AggTy = Op0->getType();
4624 Type *ValTy = I.getType();
4625 bool OutOfUndef = isa<UndefValue>(Op0);
4626
4627 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4628
4629 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4630 SmallVector<EVT, 4> ValValueVTs;
4631 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4632
4633 unsigned NumValValues = ValValueVTs.size();
4634
4635 // Ignore a extractvalue that produces an empty object
4636 if (!NumValValues) {
4637 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4638 return;
4639 }
4640
4641 SmallVector<SDValue, 4> Values(NumValValues);
4642
4643 SDValue Agg = getValue(Op0);
4644 // Copy out the selected value(s).
4645 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4646 Values[i - LinearIndex] =
4647 OutOfUndef ?
4648 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4649 SDValue(Agg.getNode(), Agg.getResNo() + i);
4650
4652 DAG.getVTList(ValValueVTs), Values));
4653}
4654
4655void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4656 Value *Op0 = I.getOperand(0);
4657 // Note that the pointer operand may be a vector of pointers. Take the scalar
4658 // element which holds a pointer.
4659 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4660 SDValue N = getValue(Op0);
4661 SDLoc dl = getCurSDLoc();
4662 auto &TLI = DAG.getTargetLoweringInfo();
4663 GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();
4664
4665 // For a vector GEP, keep the prefix scalar as long as possible, then
4666 // convert any scalars encountered after the first vector operand to vectors.
4667 bool IsVectorGEP = I.getType()->isVectorTy();
4668 ElementCount VectorElementCount =
4669 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4671
4673 GTI != E; ++GTI) {
4674 const Value *Idx = GTI.getOperand();
4675 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4676 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4677 if (Field) {
4678 // N = N + Offset
4680 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);
4681
4682 // In an inbounds GEP with an offset that is nonnegative even when
4683 // interpreted as signed, assume there is no unsigned overflow.
4684 SDNodeFlags Flags;
4685 if (NW.hasNoUnsignedWrap() ||
4686 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4688 Flags.setInBounds(NW.isInBounds());
4689
4690 N = DAG.getMemBasePlusOffset(
4691 N, DAG.getConstant(Offset, dl, N.getValueType()), dl, Flags);
4692 }
4693 } else {
4694 // IdxSize is the width of the arithmetic according to IR semantics.
4695 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4696 // (and fix up the result later).
4697 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4698 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4699 TypeSize ElementSize =
4700 GTI.getSequentialElementStride(DAG.getDataLayout());
4701 // We intentionally mask away the high bits here; ElementSize may not
4702 // fit in IdxTy.
4703 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4704 /*isSigned=*/false, /*implicitTrunc=*/true);
4705 bool ElementScalable = ElementSize.isScalable();
4706
4707 // If this is a scalar constant or a splat vector of constants,
4708 // handle it quickly.
4709 const auto *C = dyn_cast<Constant>(Idx);
4710 if (C && isa<VectorType>(C->getType()))
4711 C = C->getSplatValue();
4712
4713 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4714 if (CI && CI->isZero())
4715 continue;
4716 if (CI && !ElementScalable) {
4717 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4718 LLVMContext &Context = *DAG.getContext();
4719 SDValue OffsVal;
4720 if (N.getValueType().isVector())
4721 OffsVal = DAG.getConstant(
4722 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4723 else
4724 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4725
4726 // In an inbounds GEP with an offset that is nonnegative even when
4727 // interpreted as signed, assume there is no unsigned overflow.
4728 SDNodeFlags Flags;
4729 if (NW.hasNoUnsignedWrap() ||
4730 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4731 Flags.setNoUnsignedWrap(true);
4732 Flags.setInBounds(NW.isInBounds());
4733
4734 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4735
4736 N = DAG.getMemBasePlusOffset(N, OffsVal, dl, Flags);
4737 continue;
4738 }
4739
4740 // N = N + Idx * ElementMul;
4741 SDValue IdxN = getValue(Idx);
4742
4743 if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
4744 if (N.getValueType().isVector()) {
4745 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
4746 VectorElementCount);
4747 IdxN = DAG.getSplat(VT, dl, IdxN);
4748 } else {
4749 EVT VT =
4750 EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
4751 N = DAG.getSplat(VT, dl, N);
4752 }
4753 }
4754
4755 // If the index is smaller or larger than intptr_t, truncate or extend
4756 // it.
4757 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4758
4759 SDNodeFlags ScaleFlags;
4760 // The multiplication of an index by the type size does not wrap the
4761 // pointer index type in a signed sense (mul nsw).
4763
4764 // The multiplication of an index by the type size does not wrap the
4765 // pointer index type in an unsigned sense (mul nuw).
4766 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4767
4768 if (ElementScalable) {
4769 EVT VScaleTy = N.getValueType().getScalarType();
4770 SDValue VScale = DAG.getNode(
4771 ISD::VSCALE, dl, VScaleTy,
4772 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4773 if (N.getValueType().isVector())
4774 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4775 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale,
4776 ScaleFlags);
4777 } else {
4778 // If this is a multiply by a power of two, turn it into a shl
4779 // immediately. This is a very common case.
4780 if (ElementMul != 1) {
4781 if (ElementMul.isPowerOf2()) {
4782 unsigned Amt = ElementMul.logBase2();
4783 IdxN = DAG.getNode(
4784 ISD::SHL, dl, N.getValueType(), IdxN,
4785 DAG.getShiftAmountConstant(Amt, N.getValueType(), dl),
4786 ScaleFlags);
4787 } else {
4788 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4789 IdxN.getValueType());
4790 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, Scale,
4791 ScaleFlags);
4792 }
4793 }
4794 }
4795
4796 // The successive addition of the current address, truncated to the
4797 // pointer index type and interpreted as an unsigned number, and each
4798 // offset, also interpreted as an unsigned number, does not wrap the
4799 // pointer index type (add nuw).
4800 SDNodeFlags AddFlags;
4801 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4802 AddFlags.setInBounds(NW.isInBounds());
4803
4804 N = DAG.getMemBasePlusOffset(N, IdxN, dl, AddFlags);
4805 }
4806 }
4807
4808 if (IsVectorGEP && !N.getValueType().isVector()) {
4809 EVT VT = EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
4810 N = DAG.getSplat(VT, dl, N);
4811 }
4812
4813 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4814 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4815 if (IsVectorGEP) {
4816 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4817 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4818 }
4819
4820 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4821 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4822
4823 setValue(&I, N);
4824}
4825
4826void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4827 // If this is a fixed sized alloca in the entry block of the function,
4828 // allocate it statically on the stack.
4829 if (FuncInfo.StaticAllocaMap.count(&I))
4830 return; // getValue will auto-populate this.
4831
4832 SDLoc dl = getCurSDLoc();
4833 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4834 auto &DL = DAG.getDataLayout();
4835 TypeSize TySize = I.getAllocationBaseSize(DL);
4836 MaybeAlign Alignment = I.getAlign();
4837
4838 SDValue AllocSize = getValue(I.getArraySize());
4839
4840 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4841 if (AllocSize.getValueType() != IntPtr)
4842 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4843
4844 AllocSize = DAG.getNode(
4845 ISD::MUL, dl, IntPtr, AllocSize,
4846 DAG.getZExtOrTrunc(DAG.getTypeSize(dl, MVT::i64, TySize), dl, IntPtr));
4847
4848 // Handle alignment. If the requested alignment is less than or equal to
4849 // the stack alignment, ignore it. If the size is greater than or equal to
4850 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4851 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
4852 if (*Alignment <= StackAlign)
4853 Alignment = std::nullopt;
4854
4855 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4856 // Round the size of the allocation up to the stack alignment size
4857 // by add SA-1 to the size. This doesn't overflow because we're computing
4858 // an address inside an alloca.
4859 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4860 DAG.getConstant(StackAlignMask, dl, IntPtr),
4862
4863 // Mask out the low bits for alignment purposes.
4864 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4865 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
4866
4867 SDValue Ops[] = {
4868 getRoot(), AllocSize,
4869 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4870 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4871 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4872 setValue(&I, DSA);
4873 DAG.setRoot(DSA.getValue(1));
4874
4875 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4876}
4877
4878static const MDNode *getRangeMetadata(const Instruction &I) {
4879 return I.getMetadata(LLVMContext::MD_range);
4880}
4881
4882static std::optional<ConstantRange> getRange(const Instruction &I) {
4883 if (const auto *CB = dyn_cast<CallBase>(&I))
4884 if (std::optional<ConstantRange> CR = CB->getRange())
4885 return CR;
4886 if (const MDNode *Range = getRangeMetadata(I))
4888 return std::nullopt;
4889}
4890
4892 if (const auto *CB = dyn_cast<CallBase>(&I))
4893 return CB->getRetNoFPClass();
4894 return fcNone;
4895}
4896
4897void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4898 if (I.isAtomic())
4899 return visitAtomicLoad(I);
4900
4901 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4902 const Value *SV = I.getOperand(0);
4903 if (TLI.supportSwiftError()) {
4904 // Swifterror values can come from either a function parameter with
4905 // swifterror attribute or an alloca with swifterror attribute.
4906 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4907 if (Arg->hasSwiftErrorAttr())
4908 return visitLoadFromSwiftError(I);
4909 }
4910
4911 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4912 if (Alloca->isSwiftError())
4913 return visitLoadFromSwiftError(I);
4914 }
4915 }
4916
4917 SDValue Ptr = getValue(SV);
4918
4919 Type *Ty = I.getType();
4920 SmallVector<EVT, 4> ValueVTs, MemVTs;
4922 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4923 unsigned NumValues = ValueVTs.size();
4924 if (NumValues == 0)
4925 return;
4926
4927 Align Alignment = I.getAlign();
4928 AAMDNodes AAInfo = I.getAAMetadata();
4929 const MDNode *Ranges = getRangeMetadata(I);
4930 const MDNode *MemCacheHint = getMemCacheHint(I);
4931 bool isVolatile = I.isVolatile();
4932 MachineMemOperand::Flags MMOFlags =
4933 TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
4934
4935 SDValue Root;
4936 bool ConstantMemory = false;
4937 if (isVolatile)
4938 // Serialize volatile loads with other side effects.
4939 Root = getRoot();
4940 else if (NumValues > MaxParallelChains)
4941 Root = getMemoryRoot();
4942 else if (BatchAA &&
4943 BatchAA->pointsToConstantMemory(MemoryLocation(
4944 SV,
4945 LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4946 AAInfo))) {
4947 // Do not serialize (non-volatile) loads of constant memory with anything.
4948 Root = DAG.getEntryNode();
4949 ConstantMemory = true;
4951 } else {
4952 // Do not serialize non-volatile loads against each other.
4953 Root = DAG.getRoot();
4954 }
4955
4956 SDLoc dl = getCurSDLoc();
4957
4958 if (isVolatile)
4959 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4960
4962 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4963
4964 unsigned ChainI = 0;
4965 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4966 // Serializing loads here may result in excessive register pressure, and
4967 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4968 // could recover a bit by hoisting nodes upward in the chain by recognizing
4969 // they are side-effect free or do not alias. The optimizer should really
4970 // avoid this case by converting large object/array copies to llvm.memcpy
4971 // (MaxParallelChains should always remain as failsafe).
4972 if (ChainI == MaxParallelChains) {
4973 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4974 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4975 ArrayRef(Chains.data(), ChainI));
4976 Root = Chain;
4977 ChainI = 0;
4978 }
4979
4980 // TODO: MachinePointerInfo only supports a fixed length offset.
4981 MachinePointerInfo PtrInfo =
4982 !Offsets[i].isScalable() || Offsets[i].isZero()
4983 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4984 : MachinePointerInfo();
4985
4986 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4987 SDValue L =
4988 DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment, MMOFlags,
4989 MMOMetadata(AAInfo, Ranges, MemCacheHint));
4990 Chains[ChainI] = L.getValue(1);
4991
4992 if (MemVTs[i] != ValueVTs[i])
4993 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4994
4995 if (MDNode *NoFPClassMD = I.getMetadata(LLVMContext::MD_nofpclass)) {
4996 uint64_t FPTestInt =
4997 cast<ConstantInt>(
4998 cast<ConstantAsMetadata>(NoFPClassMD->getOperand(0))->getValue())
4999 ->getZExtValue();
5000 if (FPTestInt != fcNone) {
5001 SDValue FPTestConst =
5002 DAG.getTargetConstant(FPTestInt, SDLoc(), MVT::i32);
5003 L = DAG.getNode(ISD::AssertNoFPClass, dl, L.getValueType(), L,
5004 FPTestConst);
5005 }
5006 }
5007 Values[i] = L;
5008 }
5009
5010 if (!ConstantMemory) {
5011 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5012 ArrayRef(Chains.data(), ChainI));
5013 if (isVolatile)
5014 DAG.setRoot(Chain);
5015 else
5016 PendingLoads.push_back(Chain);
5017 }
5018
5019 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
5020 DAG.getVTList(ValueVTs), Values));
5021}
5022
5023void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
5024 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5025 "call visitStoreToSwiftError when backend supports swifterror");
5026
5027 SmallVector<EVT, 4> ValueVTs;
5028 SmallVector<uint64_t, 4> Offsets;
5029 const Value *SrcV = I.getOperand(0);
5030 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
5031 SrcV->getType(), ValueVTs, /*MemVTs=*/nullptr, &Offsets, 0);
5032 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5033 "expect a single EVT for swifterror");
5034
5035 SDValue Src = getValue(SrcV);
5036 // Create a virtual register, then update the virtual register.
5037 Register VReg =
5038 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
5039 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
5040 // Chain can be getRoot or getControlRoot.
5041 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
5042 SDValue(Src.getNode(), Src.getResNo()));
5043 DAG.setRoot(CopyNode);
5044}
5045
5046void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
5047 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
5048 "call visitLoadFromSwiftError when backend supports swifterror");
5049
5050 assert(!I.isVolatile() &&
5051 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
5052 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
5053 "Support volatile, non temporal, invariant for load_from_swift_error");
5054
5055 const Value *SV = I.getOperand(0);
5056 Type *Ty = I.getType();
5057 assert(
5058 (!BatchAA ||
5059 !BatchAA->pointsToConstantMemory(MemoryLocation(
5060 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
5061 I.getAAMetadata()))) &&
5062 "load_from_swift_error should not be constant memory");
5063
5064 SmallVector<EVT, 4> ValueVTs;
5065 SmallVector<uint64_t, 4> Offsets;
5066 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
5067 ValueVTs, /*MemVTs=*/nullptr, &Offsets, 0);
5068 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
5069 "expect a single EVT for swifterror");
5070
5071 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
5072 SDValue L = DAG.getCopyFromReg(
5073 getRoot(), getCurSDLoc(),
5074 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
5075
5076 setValue(&I, L);
5077}
5078
5079void SelectionDAGBuilder::visitStore(const StoreInst &I) {
5080 if (I.isAtomic())
5081 return visitAtomicStore(I);
5082
5083 const Value *SrcV = I.getOperand(0);
5084 const Value *PtrV = I.getOperand(1);
5085
5086 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5087 if (TLI.supportSwiftError()) {
5088 // Swifterror values can come from either a function parameter with
5089 // swifterror attribute or an alloca with swifterror attribute.
5090 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
5091 if (Arg->hasSwiftErrorAttr())
5092 return visitStoreToSwiftError(I);
5093 }
5094
5095 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
5096 if (Alloca->isSwiftError())
5097 return visitStoreToSwiftError(I);
5098 }
5099 }
5100
5101 SmallVector<EVT, 4> ValueVTs, MemVTs;
5103 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
5104 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
5105 unsigned NumValues = ValueVTs.size();
5106 if (NumValues == 0)
5107 return;
5108
5109 // Get the lowered operands. Note that we do this after
5110 // checking if NumResults is zero, because with zero results
5111 // the operands won't have values in the map.
5112 SDValue Src = getValue(SrcV);
5113 SDValue Ptr = getValue(PtrV);
5114
5115 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
5116 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
5117 SDLoc dl = getCurSDLoc();
5118 Align Alignment = I.getAlign();
5119 AAMDNodes AAInfo = I.getAAMetadata();
5120 const MDNode *MemCacheHint = getMemCacheHint(I, I.getPointerOperandIndex());
5121
5122 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
5123
5124 unsigned ChainI = 0;
5125 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
5126 // See visitLoad comments.
5127 if (ChainI == MaxParallelChains) {
5128 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5129 ArrayRef(Chains.data(), ChainI));
5130 Root = Chain;
5131 ChainI = 0;
5132 }
5133
5134 // TODO: MachinePointerInfo only supports a fixed length offset.
5135 MachinePointerInfo PtrInfo =
5136 !Offsets[i].isScalable() || Offsets[i].isZero()
5137 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
5138 : MachinePointerInfo();
5139
5140 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
5141 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
5142 if (MemVTs[i] != ValueVTs[i])
5143 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
5144 SDValue St =
5145 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags,
5146 MMOMetadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
5147 Chains[ChainI] = St;
5148 }
5149
5150 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
5151 ArrayRef(Chains.data(), ChainI));
5152 setValue(&I, StoreNode);
5153 DAG.setRoot(StoreNode);
5154}
5155
5156void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
5157 bool IsCompressing) {
5158 SDLoc sdl = getCurSDLoc();
5159
5160 Value *Src0Operand = I.getArgOperand(0);
5161 Value *PtrOperand = I.getArgOperand(1);
5162 Value *MaskOperand = I.getArgOperand(2);
5163 Align Alignment = I.getParamAlign(1).valueOrOne();
5164
5165 SDValue Ptr = getValue(PtrOperand);
5166 SDValue Src0 = getValue(Src0Operand);
5167 SDValue Mask = getValue(MaskOperand);
5168 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
5169
5170 EVT VT = Src0.getValueType();
5171
5172 const auto &TLI = DAG.getTargetLoweringInfo();
5173
5174 auto MMOFlags = MachineMemOperand::MOStore;
5175 MMOFlags |= TLI.getTargetMMOFlags(I);
5176 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5178
5179 const MDNode *MemCacheHint = getMemCacheHint(I, /*OperandNo=*/1);
5180
5181 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5182 MachinePointerInfo(PtrOperand), MMOFlags,
5183 LocationSize::upperBound(VT.getStoreSize()), Alignment,
5184 MMOMetadata(I.getAAMetadata(), /*Ranges=*/nullptr, MemCacheHint));
5185
5186 SDValue StoreNode =
5187 !IsCompressing && TTI->hasConditionalLoadStoreForType(
5188 I.getArgOperand(0)->getType(), /*IsStore=*/true)
5189 ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0,
5190 Mask)
5191 : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask,
5192 VT, MMO, ISD::UNINDEXED, /*Truncating=*/false,
5193 IsCompressing);
5194 DAG.setRoot(StoreNode);
5195 setValue(&I, StoreNode);
5196}
5197
5198// Get a uniform base for the Gather/Scatter intrinsic.
5199// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
5200// We try to represent it as a base pointer + vector of indices.
5201// Usually, the vector of pointers comes from a 'getelementptr' instruction.
5202// The first operand of the GEP may be a single pointer or a vector of pointers
5203// Example:
5204// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
5205// or
5206// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
5207// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
5208//
5209// When the first GEP operand is a single pointer - it is the uniform base we
5210// are looking for. If first operand of the GEP is a splat vector - we
5211// extract the splat value and use it as a uniform base.
5212// In all other cases the function returns 'false'.
5213static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
5214 SDValue &Scale, SelectionDAGBuilder *SDB,
5215 const BasicBlock *CurBB, uint64_t ElemSize) {
5216 SelectionDAG& DAG = SDB->DAG;
5218 const DataLayout &DL = DAG.getDataLayout();
5219
5220 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
5221
5222 // Handle splat constant pointer.
5223 if (auto *C = dyn_cast<Constant>(Ptr)) {
5224 C = C->getSplatValue();
5225 if (!C)
5226 return false;
5227
5228 Base = SDB->getValue(C);
5229
5230 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
5231 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
5232 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
5233 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
5234 return true;
5235 }
5236
5238 if (!GEP || GEP->getParent() != CurBB)
5239 return false;
5240
5241 if (GEP->getNumOperands() != 2)
5242 return false;
5243
5244 const Value *BasePtr = GEP->getPointerOperand();
5245 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
5246
5247 // Make sure the base is scalar and the index is a vector.
5248 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
5249 return false;
5250
5251 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
5252 if (ScaleVal.isScalable())
5253 return false;
5254
5255 // Target may not support the required addressing mode.
5256 if (ScaleVal != 1 &&
5257 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
5258 return false;
5259
5260 Base = SDB->getValue(BasePtr);
5261 Index = SDB->getValue(IndexVal);
5262
5263 Scale =
5264 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
5265 return true;
5266}
5267
5268void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
5269 SDLoc sdl = getCurSDLoc();
5270
5271 // llvm.masked.scatter.*(Src0, Ptrs, Mask)
5272 const Value *Ptr = I.getArgOperand(1);
5273 SDValue Src0 = getValue(I.getArgOperand(0));
5274 SDValue Mask = getValue(I.getArgOperand(2));
5275 EVT VT = Src0.getValueType();
5276 Align Alignment = I.getParamAlign(1).valueOrOne();
5277 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5278
5279 SDValue Base;
5280 SDValue Index;
5281 SDValue Scale;
5282 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
5283 I.getParent(), VT.getScalarStoreSize());
5284
5285 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5286 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5287 MachinePointerInfo(AS), MachineMemOperand::MOStore,
5288 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
5289 if (!UniformBase) {
5290 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5291 Index = getValue(Ptr);
5292 Scale =
5293 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5294 }
5295
5296 EVT IdxVT = Index.getValueType();
5297 EVT EltTy = IdxVT.getVectorElementType();
5298 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5299 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
5300 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5301 }
5302
5303 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
5304 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
5305 Ops, MMO, ISD::SIGNED_SCALED, false);
5306 DAG.setRoot(Scatter);
5307 setValue(&I, Scatter);
5308}
5309
5310void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
5311 SDLoc sdl = getCurSDLoc();
5312
5313 Value *PtrOperand = I.getArgOperand(0);
5314 Value *MaskOperand = I.getArgOperand(1);
5315 Value *Src0Operand = I.getArgOperand(2);
5316 Align Alignment = I.getParamAlign(0).valueOrOne();
5317
5318 SDValue Ptr = getValue(PtrOperand);
5319 SDValue Src0 = getValue(Src0Operand);
5320 SDValue Mask = getValue(MaskOperand);
5321 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
5322
5323 EVT VT = Src0.getValueType();
5324 AAMDNodes AAInfo = I.getAAMetadata();
5325 const MDNode *Ranges = getRangeMetadata(I);
5326 const MDNode *MemCacheHint = getMemCacheHint(I, /*OperandNo=*/0);
5327
5328 // Do not serialize masked loads of constant memory with anything.
5329 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
5330 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
5331
5332 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5333
5334 const auto &TLI = DAG.getTargetLoweringInfo();
5335
5336 auto MMOFlags = MachineMemOperand::MOLoad;
5337 MMOFlags |= TLI.getTargetMMOFlags(I);
5338 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5340 if (I.hasMetadata(LLVMContext::MD_invariant_load))
5342
5343 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5344 MachinePointerInfo(PtrOperand), MMOFlags,
5345 LocationSize::upperBound(VT.getStoreSize()), Alignment,
5346 MMOMetadata(AAInfo, Ranges, MemCacheHint));
5347
5348 // The Load/Res may point to different values and both of them are output
5349 // variables.
5350 SDValue Load;
5351 SDValue Res;
5352 if (!IsExpanding &&
5353 TTI->hasConditionalLoadStoreForType(Src0Operand->getType(),
5354 /*IsStore=*/false))
5355 Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask);
5356 else
5357 Res = Load =
5358 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
5359 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5360 if (AddToChain)
5361 PendingLoads.push_back(Load.getValue(1));
5362 setValue(&I, Res);
5363}
5364
5365void SelectionDAGBuilder::visitSpeculativeLoad(const CallInst &I) {
5366 SDLoc sdl = getCurSDLoc();
5367 Value *PtrOperand = I.getArgOperand(0);
5368 // The remaining arguments (num_accessible_bytes or oracle function + args)
5369 // are IR-level semantics only; they are not needed at codegen.
5370 SDValue Ptr = getValue(PtrOperand);
5371
5372 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5373 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5374 Align Alignment = I.getParamAlign(0).valueOrOne();
5375 AAMDNodes AAInfo = I.getAAMetadata();
5376
5377 SDValue InChain = DAG.getRoot();
5378
5379 // Use MOLoad but NOT MODereferenceable - the memory may not be
5380 // fully dereferenceable.
5381 auto MMOFlags = MachineMemOperand::MOLoad;
5382 MMOFlags |= TLI.getTargetMMOFlags(I);
5383 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5385 if (I.hasMetadata(LLVMContext::MD_invariant_load))
5387
5388 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5389 MachinePointerInfo(PtrOperand), MMOFlags,
5390 LocationSize::precise(VT.getStoreSize()), Alignment, AAInfo);
5391
5392 SDValue Load = DAG.getLoad(VT, sdl, InChain, Ptr, MMO);
5393 PendingLoads.push_back(Load.getValue(1));
5394 setValue(&I, Load);
5395}
5396
5397void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5398 SDLoc sdl = getCurSDLoc();
5399
5400 // @llvm.masked.gather.*(Ptrs, Mask, Src0)
5401 const Value *Ptr = I.getArgOperand(0);
5402 SDValue Src0 = getValue(I.getArgOperand(2));
5403 SDValue Mask = getValue(I.getArgOperand(1));
5404
5405 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5406 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5407 Align Alignment = I.getParamAlign(0).valueOrOne();
5408
5409 const MDNode *Ranges = getRangeMetadata(I);
5410
5411 SDValue Root = DAG.getRoot();
5412 SDValue Base;
5413 SDValue Index;
5414 SDValue Scale;
5415 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
5416 I.getParent(), VT.getScalarStoreSize());
5417 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5418 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5419 MachinePointerInfo(AS), MachineMemOperand::MOLoad,
5421 MMOMetadata(I.getAAMetadata(), Ranges));
5422
5423 if (!UniformBase) {
5424 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5425 Index = getValue(Ptr);
5426 Scale =
5427 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5428 }
5429
5430 EVT IdxVT = Index.getValueType();
5431 EVT EltTy = IdxVT.getVectorElementType();
5432 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5433 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
5434 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5435 }
5436
5437 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5438 SDValue Gather =
5439 DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, Ops, MMO,
5441
5442 PendingLoads.push_back(Gather.getValue(1));
5443 setValue(&I, Gather);
5444}
5445
5446void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5447 SDLoc dl = getCurSDLoc();
5448 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5449 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5450 SyncScope::ID SSID = I.getSyncScopeID();
5451
5452 SDValue InChain = getRoot();
5453
5454 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
5455 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5456
5457 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5458 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
5459
5460 MachineFunction &MF = DAG.getMachineFunction();
5461 const MDNode *MemCacheHint = getMemCacheHint(I);
5462 MachineMemOperand *MMO = MF.getMachineMemOperand(
5463 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5464 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5465 SSID, SuccessOrdering, FailureOrdering);
5466
5468 dl, MemVT, VTs, InChain,
5469 getValue(I.getPointerOperand()),
5470 getValue(I.getCompareOperand()),
5471 getValue(I.getNewValOperand()), MMO);
5472
5473 SDValue OutChain = L.getValue(2);
5474
5475 setValue(&I, L);
5476 DAG.setRoot(OutChain);
5477}
5478
5479void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5480 SDLoc dl = getCurSDLoc();
5481 ISD::NodeType NT = getAtomicRMWOpcode(I.getOperation());
5482 AtomicOrdering Ordering = I.getOrdering();
5483 SyncScope::ID SSID = I.getSyncScopeID();
5484
5485 SDValue InChain = getRoot();
5486
5487 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5488 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5489 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
5490
5491 MachineFunction &MF = DAG.getMachineFunction();
5492 const MDNode *MemCacheHint = getMemCacheHint(I);
5493 MachineMemOperand *MMO = MF.getMachineMemOperand(
5494 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5495 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5496 SSID, Ordering);
5497
5498 SDValue L =
5499 DAG.getAtomic(NT, dl, MemVT, InChain,
5500 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5501 MMO);
5502
5503 SDValue OutChain = L.getValue(1);
5504
5505 setValue(&I, L);
5506 DAG.setRoot(OutChain);
5507}
5508
5509void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5510 SDLoc dl = getCurSDLoc();
5511 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5512 SDValue Ops[3];
5513 Ops[0] = getRoot();
5514 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5515 TLI.getFenceOperandTy(DAG.getDataLayout()));
5516 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5517 TLI.getFenceOperandTy(DAG.getDataLayout()));
5518 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5519 setValue(&I, N);
5520 DAG.setRoot(N);
5521}
5522
5523void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5524 SDLoc dl = getCurSDLoc();
5525 AtomicOrdering Order = I.getOrdering();
5526 SyncScope::ID SSID = I.getSyncScopeID();
5527
5528 SDValue InChain = getRoot();
5529
5530 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5531 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5532 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5533
5534 if (!TLI.isAtomicAlignmentSupported(I.getAlign(), MemVT.getSizeInBits() / 8))
5535 report_fatal_error("Cannot generate unaligned atomic load");
5536
5537 auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
5538
5539 const MDNode *Ranges = getRangeMetadata(I);
5540 const MDNode *MemCacheHint = getMemCacheHint(I);
5541 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
5542 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5543 I.getAlign(), MMOMetadata(AAMDNodes(), Ranges, MemCacheHint), SSID,
5544 Order);
5545
5546 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5547
5548 SDValue Ptr = getValue(I.getPointerOperand());
5549 SDValue L =
5550 DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, MemVT, MemVT, InChain, Ptr, MMO);
5551
5552 SDValue OutChain = L.getValue(1);
5553 if (MemVT != VT)
5554 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5555
5556 setValue(&I, L);
5557 DAG.setRoot(OutChain);
5558}
5559
5560void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5561 SDLoc dl = getCurSDLoc();
5562
5563 AtomicOrdering Ordering = I.getOrdering();
5564 SyncScope::ID SSID = I.getSyncScopeID();
5565
5566 SDValue InChain = getRoot();
5567
5568 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5569 EVT MemVT =
5570 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5571
5572 if (!TLI.isAtomicAlignmentSupported(I.getAlign(), MemVT.getSizeInBits() / 8))
5573 report_fatal_error("Cannot generate unaligned atomic store");
5574
5575 auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
5576
5577 MachineFunction &MF = DAG.getMachineFunction();
5578 const MDNode *MemCacheHint = getMemCacheHint(I, I.getPointerOperandIndex());
5579 MachineMemOperand *MMO = MF.getMachineMemOperand(
5580 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
5581 I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
5582 SSID, Ordering);
5583
5584 SDValue Val = getValue(I.getValueOperand());
5585 if (Val.getValueType() != MemVT)
5586 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5587 SDValue Ptr = getValue(I.getPointerOperand());
5588
5589 SDValue OutChain =
5590 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5591
5592 setValue(&I, OutChain);
5593 DAG.setRoot(OutChain);
5594}
5595
5596/// Check if this intrinsic call depends on the chain (1st return value)
5597/// and if it only *loads* memory.
5598/// Ignore the callsite's attributes. A specific call site may be marked with
5599/// readnone, but the lowering code will expect the chain based on the
5600/// definition.
5601std::pair<bool, bool>
5602SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) {
5603 const Function *F = I.getCalledFunction();
5604 bool HasChain = !F->doesNotAccessMemory();
5605 bool OnlyLoad =
5606 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5607
5608 return {HasChain, OnlyLoad};
5609}
5610
5611SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
5612 const CallBase &I, bool HasChain, bool OnlyLoad,
5613 TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
5614 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5615
5616 // Build the operand list.
5618 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5619 if (OnlyLoad) {
5620 // We don't need to serialize loads against other loads.
5621 Ops.push_back(DAG.getRoot());
5622 } else {
5623 Ops.push_back(getRoot());
5624 }
5625 }
5626
5627 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5628 if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID ||
5629 TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN)
5630 Ops.push_back(DAG.getTargetConstant(I.getIntrinsicID(), getCurSDLoc(),
5631 TLI.getPointerTy(DAG.getDataLayout())));
5632
5633 // Add all operands of the call to the operand list.
5634 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5635 const Value *Arg = I.getArgOperand(i);
5636 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5637 Ops.push_back(getValue(Arg));
5638 continue;
5639 }
5640
5641 // Use TargetConstant instead of a regular constant for immarg.
5642 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5643 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5644 assert(CI->getBitWidth() <= 64 &&
5645 "large intrinsic immediates not handled");
5646 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5647 } else {
5648 Ops.push_back(
5649 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5650 }
5651 }
5652
5653 if (std::optional<OperandBundleUse> Bundle =
5654 I.getOperandBundle(LLVMContext::OB_deactivation_symbol)) {
5655 auto *Sym = Bundle->Inputs[0].get();
5656 SDValue SDSym = getValue(Sym);
5657 SDSym = DAG.getDeactivationSymbol(cast<GlobalValue>(Sym));
5658 Ops.push_back(SDSym);
5659 }
5660
5661 if (std::optional<OperandBundleUse> Bundle =
5662 I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5663 Value *Token = Bundle->Inputs[0].get();
5664 SDValue ConvControlToken = getValue(Token);
5665 assert(Ops.back().getValueType() != MVT::Glue &&
5666 "Did not expect another glue node here.");
5667 ConvControlToken =
5668 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5669 Ops.push_back(ConvControlToken);
5670 }
5671
5672 return Ops;
5673}
5674
5675SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I,
5676 bool HasChain) {
5677 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5678
5679 SmallVector<EVT, 4> ValueVTs;
5680 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5681
5682 if (HasChain)
5683 ValueVTs.push_back(MVT::Other);
5684
5685 return DAG.getVTList(ValueVTs);
5686}
5687
5688/// Get an INTRINSIC node for a target intrinsic which does not touch memory.
5689SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5690 const Type &IntrinsicVT, bool HasChain, ArrayRef<SDValue> Ops,
5691 const SDVTList &VTs) {
5692 if (!HasChain)
5693 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
5694 if (!IntrinsicVT.isVoidTy())
5695 return DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
5696 return DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
5697}
5698
5699/// Set root, convert return type if necessary and check alignment.
5700SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I,
5701 bool HasChain,
5702 bool OnlyLoad,
5703 SDValue Result) {
5704 if (HasChain) {
5705 SDValue Chain = Result.getValue(Result.getNode()->getNumValues() - 1);
5706 if (OnlyLoad)
5707 PendingLoads.push_back(Chain);
5708 else
5709 DAG.setRoot(Chain);
5710 }
5711
5712 if (I.getType()->isVoidTy())
5713 return Result;
5714
5715 if (MaybeAlign Alignment = I.getRetAlign(); InsertAssertAlign && Alignment) {
5716 // Insert `assertalign` node if there's an alignment.
5717 Result = DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5718 } else if (!isa<VectorType>(I.getType())) {
5719 Result = lowerRangeToAssertZExt(DAG, I, Result);
5720 }
5721
5722 return Result;
5723}
5724
5725/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5726/// node.
5727void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5728 unsigned Intrinsic) {
5729 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I);
5730 Intrinsic::ID IntrinsicID = static_cast<Intrinsic::ID>(Intrinsic);
5731
5732 if (!DAG.getMachineFunction().getSubtarget().isIntrinsicSupported(
5733 Intrinsic)) {
5734 SDLoc DL = getCurSDLoc();
5735 DAG.getContext()->diagnose(DiagnosticInfoUnsupportedTargetIntrinsic(
5736 *I.getFunction(), IntrinsicID, DL.getDebugLoc()));
5737
5738 // The intrinsic is not available on this subtarget. Preserve the chain for
5739 // side-effecting intrinsics and lower any result to poison so that
5740 // compilation can continue and collect further diagnostics.
5741 if (HasChain && !OnlyLoad)
5742 DAG.setRoot(getRoot());
5743
5745 return;
5746 }
5747
5748 // Infos is set by getTgtMemIntrinsic.
5750 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5751 TLI.getTgtMemIntrinsic(Infos, I, DAG.getMachineFunction(), Intrinsic);
5752 // Use the first (primary) info determines the node opcode.
5753 TargetLowering::IntrinsicInfo *Info = !Infos.empty() ? &Infos[0] : nullptr;
5754
5756 getTargetIntrinsicOperands(I, HasChain, OnlyLoad, Info);
5757 SDVTList VTs = getTargetIntrinsicVTList(I, HasChain);
5758
5759 // Propagate fast-math-flags from IR to node(s).
5760 SDNodeFlags Flags;
5761 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5762 Flags.copyFMF(*FPMO);
5763 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5764
5765 // Create the node.
5767
5768 // In some cases, custom collection of operands from CallInst I may be needed.
5770 if (!Infos.empty()) {
5771 // This is target intrinsic that touches memory
5772 // Create MachineMemOperands for each memory access described by the target.
5773 MachineFunction &MF = DAG.getMachineFunction();
5775 for (const auto &Info : Infos) {
5776 // TODO: We currently just fallback to address space 0 if
5777 // getTgtMemIntrinsic didn't yield anything useful.
5778 MachinePointerInfo MPI;
5779 if (Info.ptrVal)
5780 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5781 else if (Info.fallbackAddressSpace)
5782 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5783 EVT MemVT = Info.memVT;
5784 LocationSize Size = LocationSize::precise(Info.size);
5785 if (Size.hasValue() && !Size.getValue())
5787 Align Alignment = Info.align.value_or(DAG.getEVTAlign(MemVT));
5788 MachineMemOperand *MMO = MF.getMachineMemOperand(
5789 MPI, Info.flags, Size, Alignment, I.getAAMetadata(), Info.ssid,
5790 Info.order, Info.failureOrder);
5791 MMOs.push_back(MMO);
5792 }
5793
5794 Result = DAG.getMemIntrinsicNode(Info->opc, getCurSDLoc(), VTs, Ops,
5795 Info->memVT, MMOs);
5796 } else {
5797 Result = getTargetNonMemIntrinsicNode(*I.getType(), HasChain, Ops, VTs);
5798 }
5799
5800 Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result);
5801
5802 setValue(&I, Result);
5803}
5804
5805/// GetSignificand - Get the significand and build it into a floating-point
5806/// number with exponent of 1:
5807///
5808/// Op = (Op & 0x007fffff) | 0x3f800000;
5809///
5810/// where Op is the hexadecimal representation of floating point value.
5812 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5813 DAG.getConstant(0x007fffff, dl, MVT::i32));
5814 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5815 DAG.getConstant(0x3f800000, dl, MVT::i32));
5816 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5817}
5818
5819/// GetExponent - Get the exponent:
5820///
5821/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5822///
5823/// where Op is the hexadecimal representation of floating point value.
5825 const TargetLowering &TLI, const SDLoc &dl) {
5826 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5827 DAG.getConstant(0x7f800000, dl, MVT::i32));
5828 SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
5829 DAG.getShiftAmountConstant(23, MVT::i32, dl));
5830 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5831 DAG.getConstant(127, dl, MVT::i32));
5832 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5833}
5834
5835/// getF32Constant - Get 32-bit floating point constant.
5836static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
5837 const SDLoc &dl) {
5838 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5839 MVT::f32);
5840}
5841
5843 SelectionDAG &DAG) {
5844 // TODO: What fast-math-flags should be set on the floating-point nodes?
5845
5846 // IntegerPartOfX = ((int32_t)(t0);
5847 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5848
5849 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5850 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5851 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5852
5853 // IntegerPartOfX <<= 23;
5854 IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5855 DAG.getShiftAmountConstant(23, MVT::i32, dl));
5856
5857 SDValue TwoToFractionalPartOfX;
5858 if (LimitFloatPrecision <= 6) {
5859 // For floating-point precision of 6:
5860 //
5861 // TwoToFractionalPartOfX =
5862 // 0.997535578f +
5863 // (0.735607626f + 0.252464424f * x) * x;
5864 //
5865 // error 0.0144103317, which is 6 bits
5866 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5867 getF32Constant(DAG, 0x3e814304, dl));
5868 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5869 getF32Constant(DAG, 0x3f3c50c8, dl));
5870 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5871 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5872 getF32Constant(DAG, 0x3f7f5e7e, dl));
5873 } else if (LimitFloatPrecision <= 12) {
5874 // For floating-point precision of 12:
5875 //
5876 // TwoToFractionalPartOfX =
5877 // 0.999892986f +
5878 // (0.696457318f +
5879 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5880 //
5881 // error 0.000107046256, which is 13 to 14 bits
5882 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5883 getF32Constant(DAG, 0x3da235e3, dl));
5884 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5885 getF32Constant(DAG, 0x3e65b8f3, dl));
5886 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5887 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5888 getF32Constant(DAG, 0x3f324b07, dl));
5889 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5890 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5891 getF32Constant(DAG, 0x3f7ff8fd, dl));
5892 } else { // LimitFloatPrecision <= 18
5893 // For floating-point precision of 18:
5894 //
5895 // TwoToFractionalPartOfX =
5896 // 0.999999982f +
5897 // (0.693148872f +
5898 // (0.240227044f +
5899 // (0.554906021e-1f +
5900 // (0.961591928e-2f +
5901 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5902 // error 2.47208000*10^(-7), which is better than 18 bits
5903 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5904 getF32Constant(DAG, 0x3924b03e, dl));
5905 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5906 getF32Constant(DAG, 0x3ab24b87, dl));
5907 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5908 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5909 getF32Constant(DAG, 0x3c1d8c17, dl));
5910 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5911 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5912 getF32Constant(DAG, 0x3d634a1d, dl));
5913 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5914 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5915 getF32Constant(DAG, 0x3e75fe14, dl));
5916 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5917 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5918 getF32Constant(DAG, 0x3f317234, dl));
5919 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5920 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5921 getF32Constant(DAG, 0x3f800000, dl));
5922 }
5923
5924 // Add the exponent into the result in integer domain.
5925 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5926 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5927 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5928}
5929
5930/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5931/// limited-precision mode.
5933 const TargetLowering &TLI, SDNodeFlags Flags) {
5934 if (Op.getValueType() == MVT::f32 &&
5936
5937 // Put the exponent in the right bit position for later addition to the
5938 // final result:
5939 //
5940 // t0 = Op * log2(e)
5941
5942 // TODO: What fast-math-flags should be set here?
5943 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5944 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5945 return getLimitedPrecisionExp2(t0, dl, DAG);
5946 }
5947
5948 // No special expansion.
5949 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5950}
5951
5952/// expandLog - Lower a log intrinsic. Handles the special sequences for
5953/// limited-precision mode.
5955 const TargetLowering &TLI, SDNodeFlags Flags) {
5956 // TODO: What fast-math-flags should be set on the floating-point nodes?
5957
5958 if (Op.getValueType() == MVT::f32 &&
5960 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5961
5962 // Scale the exponent by log(2).
5963 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5964 SDValue LogOfExponent =
5965 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5966 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5967
5968 // Get the significand and build it into a floating-point number with
5969 // exponent of 1.
5970 SDValue X = GetSignificand(DAG, Op1, dl);
5971
5972 SDValue LogOfMantissa;
5973 if (LimitFloatPrecision <= 6) {
5974 // For floating-point precision of 6:
5975 //
5976 // LogofMantissa =
5977 // -1.1609546f +
5978 // (1.4034025f - 0.23903021f * x) * x;
5979 //
5980 // error 0.0034276066, which is better than 8 bits
5981 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5982 getF32Constant(DAG, 0xbe74c456, dl));
5983 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5984 getF32Constant(DAG, 0x3fb3a2b1, dl));
5985 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5986 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5987 getF32Constant(DAG, 0x3f949a29, dl));
5988 } else if (LimitFloatPrecision <= 12) {
5989 // For floating-point precision of 12:
5990 //
5991 // LogOfMantissa =
5992 // -1.7417939f +
5993 // (2.8212026f +
5994 // (-1.4699568f +
5995 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5996 //
5997 // error 0.000061011436, which is 14 bits
5998 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5999 getF32Constant(DAG, 0xbd67b6d6, dl));
6000 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6001 getF32Constant(DAG, 0x3ee4f4b8, dl));
6002 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6003 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6004 getF32Constant(DAG, 0x3fbc278b, dl));
6005 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6006 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6007 getF32Constant(DAG, 0x40348e95, dl));
6008 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6009 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6010 getF32Constant(DAG, 0x3fdef31a, dl));
6011 } else { // LimitFloatPrecision <= 18
6012 // For floating-point precision of 18:
6013 //
6014 // LogOfMantissa =
6015 // -2.1072184f +
6016 // (4.2372794f +
6017 // (-3.7029485f +
6018 // (2.2781945f +
6019 // (-0.87823314f +
6020 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
6021 //
6022 // error 0.0000023660568, which is better than 18 bits
6023 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6024 getF32Constant(DAG, 0xbc91e5ac, dl));
6025 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6026 getF32Constant(DAG, 0x3e4350aa, dl));
6027 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6028 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6029 getF32Constant(DAG, 0x3f60d3e3, dl));
6030 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6031 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6032 getF32Constant(DAG, 0x4011cdf0, dl));
6033 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6034 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6035 getF32Constant(DAG, 0x406cfd1c, dl));
6036 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6037 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
6038 getF32Constant(DAG, 0x408797cb, dl));
6039 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
6040 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
6041 getF32Constant(DAG, 0x4006dcab, dl));
6042 }
6043
6044 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
6045 }
6046
6047 // No special expansion.
6048 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
6049}
6050
6051/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
6052/// limited-precision mode.
6054 const TargetLowering &TLI, SDNodeFlags Flags) {
6055 // TODO: What fast-math-flags should be set on the floating-point nodes?
6056
6057 if (Op.getValueType() == MVT::f32 &&
6059 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
6060
6061 // Get the exponent.
6062 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
6063
6064 // Get the significand and build it into a floating-point number with
6065 // exponent of 1.
6066 SDValue X = GetSignificand(DAG, Op1, dl);
6067
6068 // Different possible minimax approximations of significand in
6069 // floating-point for various degrees of accuracy over [1,2].
6070 SDValue Log2ofMantissa;
6071 if (LimitFloatPrecision <= 6) {
6072 // For floating-point precision of 6:
6073 //
6074 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
6075 //
6076 // error 0.0049451742, which is more than 7 bits
6077 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6078 getF32Constant(DAG, 0xbeb08fe0, dl));
6079 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6080 getF32Constant(DAG, 0x40019463, dl));
6081 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6082 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6083 getF32Constant(DAG, 0x3fd6633d, dl));
6084 } else if (LimitFloatPrecision <= 12) {
6085 // For floating-point precision of 12:
6086 //
6087 // Log2ofMantissa =
6088 // -2.51285454f +
6089 // (4.07009056f +
6090 // (-2.12067489f +
6091 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
6092 //
6093 // error 0.0000876136000, which is better than 13 bits
6094 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6095 getF32Constant(DAG, 0xbda7262e, dl));
6096 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6097 getF32Constant(DAG, 0x3f25280b, dl));
6098 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6099 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6100 getF32Constant(DAG, 0x4007b923, dl));
6101 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6102 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6103 getF32Constant(DAG, 0x40823e2f, dl));
6104 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6105 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6106 getF32Constant(DAG, 0x4020d29c, dl));
6107 } else { // LimitFloatPrecision <= 18
6108 // For floating-point precision of 18:
6109 //
6110 // Log2ofMantissa =
6111 // -3.0400495f +
6112 // (6.1129976f +
6113 // (-5.3420409f +
6114 // (3.2865683f +
6115 // (-1.2669343f +
6116 // (0.27515199f -
6117 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
6118 //
6119 // error 0.0000018516, which is better than 18 bits
6120 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6121 getF32Constant(DAG, 0xbcd2769e, dl));
6122 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6123 getF32Constant(DAG, 0x3e8ce0b9, dl));
6124 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6125 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6126 getF32Constant(DAG, 0x3fa22ae7, dl));
6127 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6128 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
6129 getF32Constant(DAG, 0x40525723, dl));
6130 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6131 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
6132 getF32Constant(DAG, 0x40aaf200, dl));
6133 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6134 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
6135 getF32Constant(DAG, 0x40c39dad, dl));
6136 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
6137 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
6138 getF32Constant(DAG, 0x4042902c, dl));
6139 }
6140
6141 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
6142 }
6143
6144 // No special expansion.
6145 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
6146}
6147
6148/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
6149/// limited-precision mode.
6151 const TargetLowering &TLI, SDNodeFlags Flags) {
6152 // TODO: What fast-math-flags should be set on the floating-point nodes?
6153
6154 if (Op.getValueType() == MVT::f32 &&
6156 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
6157
6158 // Scale the exponent by log10(2) [0.30102999f].
6159 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
6160 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
6161 getF32Constant(DAG, 0x3e9a209a, dl));
6162
6163 // Get the significand and build it into a floating-point number with
6164 // exponent of 1.
6165 SDValue X = GetSignificand(DAG, Op1, dl);
6166
6167 SDValue Log10ofMantissa;
6168 if (LimitFloatPrecision <= 6) {
6169 // For floating-point precision of 6:
6170 //
6171 // Log10ofMantissa =
6172 // -0.50419619f +
6173 // (0.60948995f - 0.10380950f * x) * x;
6174 //
6175 // error 0.0014886165, which is 6 bits
6176 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6177 getF32Constant(DAG, 0xbdd49a13, dl));
6178 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
6179 getF32Constant(DAG, 0x3f1c0789, dl));
6180 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6181 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
6182 getF32Constant(DAG, 0x3f011300, dl));
6183 } else if (LimitFloatPrecision <= 12) {
6184 // For floating-point precision of 12:
6185 //
6186 // Log10ofMantissa =
6187 // -0.64831180f +
6188 // (0.91751397f +
6189 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
6190 //
6191 // error 0.00019228036, which is better than 12 bits
6192 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6193 getF32Constant(DAG, 0x3d431f31, dl));
6194 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
6195 getF32Constant(DAG, 0x3ea21fb2, dl));
6196 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6197 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
6198 getF32Constant(DAG, 0x3f6ae232, dl));
6199 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6200 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
6201 getF32Constant(DAG, 0x3f25f7c3, dl));
6202 } else { // LimitFloatPrecision <= 18
6203 // For floating-point precision of 18:
6204 //
6205 // Log10ofMantissa =
6206 // -0.84299375f +
6207 // (1.5327582f +
6208 // (-1.0688956f +
6209 // (0.49102474f +
6210 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
6211 //
6212 // error 0.0000037995730, which is better than 18 bits
6213 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
6214 getF32Constant(DAG, 0x3c5d51ce, dl));
6215 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
6216 getF32Constant(DAG, 0x3e00685a, dl));
6217 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
6218 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
6219 getF32Constant(DAG, 0x3efb6798, dl));
6220 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
6221 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
6222 getF32Constant(DAG, 0x3f88d192, dl));
6223 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
6224 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
6225 getF32Constant(DAG, 0x3fc4316c, dl));
6226 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
6227 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
6228 getF32Constant(DAG, 0x3f57ce70, dl));
6229 }
6230
6231 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
6232 }
6233
6234 // No special expansion.
6235 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
6236}
6237
6238/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
6239/// limited-precision mode.
6241 const TargetLowering &TLI, SDNodeFlags Flags) {
6242 if (Op.getValueType() == MVT::f32 &&
6244 return getLimitedPrecisionExp2(Op, dl, DAG);
6245
6246 // No special expansion.
6247 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
6248}
6249
6250/// visitPow - Lower a pow intrinsic. Handles the special sequences for
6251/// limited-precision mode with x == 10.0f.
6253 SelectionDAG &DAG, const TargetLowering &TLI,
6254 SDNodeFlags Flags) {
6255 bool IsExp10 = false;
6256 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
6259 APFloat Ten(10.0f);
6260 IsExp10 = LHSC->isExactlyValue(Ten);
6261 }
6262 }
6263
6264 // TODO: What fast-math-flags should be set on the FMUL node?
6265 if (IsExp10) {
6266 // Put the exponent in the right bit position for later addition to the
6267 // final result:
6268 //
6269 // #define LOG2OF10 3.3219281f
6270 // t0 = Op * LOG2OF10;
6271 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
6272 getF32Constant(DAG, 0x40549a78, dl));
6273 return getLimitedPrecisionExp2(t0, dl, DAG);
6274 }
6275
6276 // No special expansion.
6277 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
6278}
6279
6280/// ExpandPowI - Expand a llvm.powi intrinsic.
6282 SelectionDAG &DAG) {
6283 // If RHS is a constant, we can expand this out to a multiplication tree if
6284 // it's beneficial on the target, otherwise we end up lowering to a call to
6285 // __powidf2 (for example).
6287 unsigned Val = RHSC->getSExtValue();
6288
6289 // powi(x, 0) -> 1.0
6290 if (Val == 0)
6291 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
6292
6294 Val, DAG.shouldOptForSize())) {
6295 // Get the exponent as a positive value.
6296 if ((int)Val < 0)
6297 Val = -Val;
6298 // We use the simple binary decomposition method to generate the multiply
6299 // sequence. There are more optimal ways to do this (for example,
6300 // powi(x,15) generates one more multiply than it should), but this has
6301 // the benefit of being both really simple and much better than a libcall.
6302 SDValue Res; // Logically starts equal to 1.0
6303 SDValue CurSquare = LHS;
6304 // TODO: Intrinsics should have fast-math-flags that propagate to these
6305 // nodes.
6306 while (Val) {
6307 if (Val & 1) {
6308 if (Res.getNode())
6309 Res =
6310 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
6311 else
6312 Res = CurSquare; // 1.0*CurSquare.
6313 }
6314
6315 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
6316 CurSquare, CurSquare);
6317 Val >>= 1;
6318 }
6319
6320 // If the original was negative, invert the result, producing 1/(x*x*x).
6321 if (RHSC->getSExtValue() < 0)
6322 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
6323 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
6324 return Res;
6325 }
6326 }
6327
6328 // Otherwise, expand to a libcall.
6329 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
6330}
6331
6332static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
6333 SDValue LHS, SDValue RHS, SDValue Scale,
6334 SelectionDAG &DAG, const TargetLowering &TLI) {
6335 EVT VT = LHS.getValueType();
6336 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
6337 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
6338 LLVMContext &Ctx = *DAG.getContext();
6339
6340 // If the type is legal but the operation isn't, this node might survive all
6341 // the way to operation legalization. If we end up there and we do not have
6342 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
6343 // node.
6344
6345 // Coax the legalizer into expanding the node during type legalization instead
6346 // by bumping the size by one bit. This will force it to Promote, enabling the
6347 // early expansion and avoiding the need to expand later.
6348
6349 // We don't have to do this if Scale is 0; that can always be expanded, unless
6350 // it's a saturating signed operation. Those can experience true integer
6351 // division overflow, a case which we must avoid.
6352
6353 // FIXME: We wouldn't have to do this (or any of the early
6354 // expansion/promotion) if it was possible to expand a libcall of an
6355 // illegal type during operation legalization. But it's not, so things
6356 // get a bit hacky.
6357 unsigned ScaleInt = Scale->getAsZExtVal();
6358 if ((ScaleInt > 0 || (Saturating && Signed)) &&
6359 (TLI.isTypeLegal(VT) ||
6360 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
6361 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
6362 Opcode, VT, ScaleInt);
6363 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
6364 EVT PromVT;
6365 if (VT.isScalarInteger())
6366 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
6367 else if (VT.isVector()) {
6368 PromVT = VT.getVectorElementType();
6369 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
6370 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
6371 } else
6372 llvm_unreachable("Wrong VT for DIVFIX?");
6373 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
6374 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
6375 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
6376 // For saturating operations, we need to shift up the LHS to get the
6377 // proper saturation width, and then shift down again afterwards.
6378 if (Saturating)
6379 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
6380 DAG.getConstant(1, DL, ShiftTy));
6381 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
6382 if (Saturating)
6383 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
6384 DAG.getConstant(1, DL, ShiftTy));
6385 return DAG.getZExtOrTrunc(Res, DL, VT);
6386 }
6387 }
6388
6389 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
6390}
6391
6392// getUnderlyingArgRegs - Find underlying registers used for a truncated,
6393// bitcasted, or split argument. Returns a list of <Register, size in bits>
6394static void
6395getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
6396 const SDValue &N) {
6397 switch (N.getOpcode()) {
6398 case ISD::CopyFromReg: {
6399 SDValue Op = N.getOperand(1);
6400 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
6401 Op.getValueType().getSizeInBits());
6402 return;
6403 }
6404 case ISD::BITCAST:
6405 case ISD::AssertZext:
6406 case ISD::AssertSext:
6407 case ISD::TRUNCATE:
6408 getUnderlyingArgRegs(Regs, N.getOperand(0));
6409 return;
6410 case ISD::BUILD_PAIR:
6411 case ISD::BUILD_VECTOR:
6413 for (SDValue Op : N->op_values())
6414 getUnderlyingArgRegs(Regs, Op);
6415 return;
6416 default:
6417 return;
6418 }
6419}
6420
6421/// If the DbgValueInst is a dbg_value of a function argument, create the
6422/// corresponding DBG_VALUE machine instruction for it now. At the end of
6423/// instruction selection, they will be inserted to the entry BB.
6424/// We don't currently support this for variadic dbg_values, as they shouldn't
6425/// appear for function arguments or in the prologue.
6426bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6427 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6428 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6429 const Argument *Arg = dyn_cast<Argument>(V);
6430 if (!Arg)
6431 return false;
6432
6433 MachineFunction &MF = DAG.getMachineFunction();
6434 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6435
6436 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6437 // we've been asked to pursue.
6438 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6439 bool Indirect) {
6440 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6441 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6442 // pointing at the VReg, which will be patched up later.
6443 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
6445 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6446 /* isKill */ false, /* isDead */ false,
6447 /* isUndef */ false, /* isEarlyClobber */ false,
6448 /* SubReg */ 0, /* isDebug */ true)});
6449
6450 auto *NewDIExpr = FragExpr;
6451 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6452 // the DIExpression.
6453 if (Indirect)
6454 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
6456 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
6457 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
6458 } else {
6459 // Create a completely standard DBG_VALUE.
6460 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
6461 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
6462 }
6463 };
6464
6465 if (Kind == FuncArgumentDbgValueKind::Value) {
6466 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6467 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6468 // the entry block.
6469 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6470 if (!IsInEntryBlock)
6471 return false;
6472
6473 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6474 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6475 // variable that also is a param.
6476 //
6477 // Although, if we are at the top of the entry block already, we can still
6478 // emit using ArgDbgValue. This might catch some situations when the
6479 // dbg.value refers to an argument that isn't used in the entry block, so
6480 // any CopyToReg node would be optimized out and the only way to express
6481 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6482 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6483 // we should only emit as ArgDbgValue if the Variable is an argument to the
6484 // current function, and the dbg.value intrinsic is found in the entry
6485 // block.
6486 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6487 !DL->getInlinedAt();
6488 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6489 if (!IsInPrologue && !VariableIsFunctionInputArg)
6490 return false;
6491
6492 // Here we assume that a function argument on IR level only can be used to
6493 // describe one input parameter on source level. If we for example have
6494 // source code like this
6495 //
6496 // struct A { long x, y; };
6497 // void foo(struct A a, long b) {
6498 // ...
6499 // b = a.x;
6500 // ...
6501 // }
6502 //
6503 // and IR like this
6504 //
6505 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6506 // entry:
6507 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6508 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6509 // call void @llvm.dbg.value(metadata i32 %b, "b",
6510 // ...
6511 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6512 // ...
6513 //
6514 // then the last dbg.value is describing a parameter "b" using a value that
6515 // is an argument. But since we already has used %a1 to describe a parameter
6516 // we should not handle that last dbg.value here (that would result in an
6517 // incorrect hoisting of the DBG_VALUE to the function entry).
6518 // Notice that we allow one dbg.value per IR level argument, to accommodate
6519 // for the situation with fragments above.
6520 // If there is no node for the value being handled, we return true to skip
6521 // the normal generation of debug info, as it would kill existing debug
6522 // info for the parameter in case of duplicates.
6523 if (VariableIsFunctionInputArg) {
6524 unsigned ArgNo = Arg->getArgNo();
6525 if (ArgNo >= FuncInfo.DescribedArgs.size())
6526 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6527 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6528 return !NodeMap[V].getNode();
6529 FuncInfo.DescribedArgs.set(ArgNo);
6530 }
6531 }
6532
6533 bool IsIndirect = false;
6534 std::optional<MachineOperand> Op;
6535 // Some arguments' frame index is recorded during argument lowering.
6536 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6537 if (FI != std::numeric_limits<int>::max())
6539
6541 if (!Op && N.getNode()) {
6542 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6543 Register Reg;
6544 if (ArgRegsAndSizes.size() == 1)
6545 Reg = ArgRegsAndSizes.front().first;
6546
6547 if (Reg && Reg.isVirtual()) {
6548 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6549 Register PR = RegInfo.getLiveInPhysReg(Reg);
6550 if (PR)
6551 Reg = PR;
6552 }
6553 if (Reg) {
6555 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6556 }
6557 }
6558
6559 if (!Op && N.getNode()) {
6560 // Check if frame index is available.
6561 SDValue LCandidate = peekThroughBitcasts(N);
6562 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6563 if (FrameIndexSDNode *FINode =
6564 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6565 Op = MachineOperand::CreateFI(FINode->getIndex());
6566 }
6567
6568 if (!Op) {
6569 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6570 auto splitMultiRegDbgValue =
6571 [&](ArrayRef<std::pair<Register, TypeSize>> SplitRegs) -> bool {
6572 unsigned Offset = 0;
6573 for (const auto &[Reg, RegSizeInBits] : SplitRegs) {
6574 // FIXME: Scalable sizes are not supported in fragment expressions.
6575 if (RegSizeInBits.isScalable())
6576 return false;
6577
6578 // If the expression is already a fragment, the current register
6579 // offset+size might extend beyond the fragment. In this case, only
6580 // the register bits that are inside the fragment are relevant.
6581 int RegFragmentSizeInBits = RegSizeInBits.getFixedValue();
6582 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6583 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6584 // The register is entirely outside the expression fragment,
6585 // so is irrelevant for debug info.
6586 if (Offset >= ExprFragmentSizeInBits)
6587 break;
6588 // The register is partially outside the expression fragment, only
6589 // the low bits within the fragment are relevant for debug info.
6590 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6591 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6592 }
6593 }
6594
6595 auto FragmentExpr = DIExpression::createFragmentExpression(
6596 Expr, Offset, RegFragmentSizeInBits);
6597 Offset += RegSizeInBits.getFixedValue();
6598 // If a valid fragment expression cannot be created, the variable's
6599 // correct value cannot be determined and so it is set as poison.
6600 if (!FragmentExpr) {
6601 SDDbgValue *SDV = DAG.getConstantDbgValue(
6602 Variable, Expr, PoisonValue::get(V->getType()), DL, SDNodeOrder);
6603 DAG.AddDbgValue(SDV, false);
6604 continue;
6605 }
6606 MachineInstr *NewMI = MakeVRegDbgValue(
6607 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6608 FuncInfo.ArgDbgValues.push_back(NewMI);
6609 }
6610
6611 return true;
6612 };
6613
6614 // Check if ValueMap has reg number.
6616 VMI = FuncInfo.ValueMap.find(V);
6617 if (VMI != FuncInfo.ValueMap.end()) {
6618 const auto &TLI = DAG.getTargetLoweringInfo();
6619 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6620 V->getType(), std::nullopt);
6621 if (RFV.occupiesMultipleRegs())
6622 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6623
6624 Op = MachineOperand::CreateReg(VMI->second, false);
6625 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6626 } else if (ArgRegsAndSizes.size() > 1) {
6627 // This was split due to the calling convention, and no virtual register
6628 // mapping exists for the value.
6629 return splitMultiRegDbgValue(ArgRegsAndSizes);
6630 }
6631 }
6632
6633 if (!Op)
6634 return false;
6635
6636 assert(Variable->isValidLocationForIntrinsic(DL) &&
6637 "Expected inlined-at fields to agree");
6638 MachineInstr *NewMI = nullptr;
6639
6640 if (Op->isReg())
6641 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6642 else
6643 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6644 Variable, Expr);
6645
6646 // Otherwise, use ArgDbgValues.
6647 FuncInfo.ArgDbgValues.push_back(NewMI);
6648 return true;
6649}
6650
6651/// Return the appropriate SDDbgValue based on N.
6652SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6653 DILocalVariable *Variable,
6654 DIExpression *Expr,
6655 const DebugLoc &dl,
6656 unsigned DbgSDNodeOrder) {
6657 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6658 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6659 // stack slot locations.
6660 //
6661 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6662 // debug values here after optimization:
6663 //
6664 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6665 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6666 //
6667 // Both describe the direct values of their associated variables.
6668 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6669 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6670 }
6671 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6672 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6673}
6674
6675static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6676 switch (Intrinsic) {
6677 case Intrinsic::smul_fix:
6678 return ISD::SMULFIX;
6679 case Intrinsic::umul_fix:
6680 return ISD::UMULFIX;
6681 case Intrinsic::smul_fix_sat:
6682 return ISD::SMULFIXSAT;
6683 case Intrinsic::umul_fix_sat:
6684 return ISD::UMULFIXSAT;
6685 case Intrinsic::sdiv_fix:
6686 return ISD::SDIVFIX;
6687 case Intrinsic::udiv_fix:
6688 return ISD::UDIVFIX;
6689 case Intrinsic::sdiv_fix_sat:
6690 return ISD::SDIVFIXSAT;
6691 case Intrinsic::udiv_fix_sat:
6692 return ISD::UDIVFIXSAT;
6693 default:
6694 llvm_unreachable("Unhandled fixed point intrinsic");
6695 }
6696}
6697
6698/// Given a @llvm.call.preallocated.setup, return the corresponding
6699/// preallocated call.
6700static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6701 assert(cast<CallBase>(PreallocatedSetup)
6703 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6704 "expected call_preallocated_setup Value");
6705 for (const auto *U : PreallocatedSetup->users()) {
6706 auto *UseCall = cast<CallBase>(U);
6707 const Function *Fn = UseCall->getCalledFunction();
6708 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6709 return UseCall;
6710 }
6711 }
6712 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6713}
6714
6715/// If DI is a debug value with an EntryValue expression, lower it using the
6716/// corresponding physical register of the associated Argument value
6717/// (guaranteed to exist by the verifier).
6718bool SelectionDAGBuilder::visitEntryValueDbgValue(
6720 DIExpression *Expr, DebugLoc DbgLoc) {
6721 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6722 return false;
6723
6724 // These properties are guaranteed by the verifier.
6725 const Argument *Arg = cast<Argument>(Values[0]);
6726 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6727
6728 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6729 if (ArgIt == FuncInfo.ValueMap.end()) {
6730 LLVM_DEBUG(
6731 dbgs() << "Dropping dbg.value: expression is entry_value but "
6732 "couldn't find an associated register for the Argument\n");
6733 return true;
6734 }
6735 Register ArgVReg = ArgIt->getSecond();
6736
6737 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6738 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6739 SDDbgValue *SDV = DAG.getVRegDbgValue(
6740 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6741 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6742 return true;
6743 }
6744 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6745 "couldn't find a physical register\n");
6746 return true;
6747}
6748
6749/// Lower the call to the specified intrinsic function.
6750void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6751 unsigned Intrinsic) {
6752 SDLoc sdl = getCurSDLoc();
6753 switch (Intrinsic) {
6754 case Intrinsic::experimental_convergence_anchor:
6755 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6756 break;
6757 case Intrinsic::experimental_convergence_entry:
6758 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6759 break;
6760 case Intrinsic::experimental_convergence_loop: {
6761 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6762 auto *Token = Bundle->Inputs[0].get();
6763 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6764 getValue(Token)));
6765 break;
6766 }
6767 }
6768}
6769
6770void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6771 unsigned IntrinsicID) {
6772 // For now, we're only lowering an 'add' histogram.
6773 // We can add others later, e.g. saturating adds, min/max.
6774 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6775 "Tried to lower unsupported histogram type");
6776 SDLoc sdl = getCurSDLoc();
6777 Value *Ptr = I.getOperand(0);
6778 SDValue Inc = getValue(I.getOperand(1));
6779 SDValue Mask = getValue(I.getOperand(2));
6780
6781 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6782 EVT VT = Inc.getValueType();
6783 Align Alignment = DAG.getEVTAlign(VT);
6784
6785 const MDNode *Ranges = getRangeMetadata(I);
6786
6787 SDValue Root = DAG.getRoot();
6788 SDValue Base;
6789 SDValue Index;
6790 SDValue Scale;
6791 bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
6792 I.getParent(), VT.getScalarStoreSize());
6793
6794 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6795
6796 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
6797 MachinePointerInfo(AS),
6799 MemoryLocation::UnknownSize, Alignment,
6800 MMOMetadata(I.getAAMetadata(), Ranges));
6801
6802 if (!UniformBase) {
6803 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
6804 Index = getValue(Ptr);
6805 Scale =
6806 DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
6807 }
6808
6809 EVT IdxVT = Index.getValueType();
6810
6811 // Avoid using e.g. i32 as index type when the increment must be performed
6812 // on i64's.
6813 bool MustExtendIndex = VT.getScalarSizeInBits() > IdxVT.getScalarSizeInBits();
6814 EVT EltTy = MustExtendIndex ? VT : IdxVT.getVectorElementType();
6815 if (MustExtendIndex || TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
6816 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
6817 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
6818 }
6819
6820 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6821
6822 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6823 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
6824 Ops, MMO, ISD::SIGNED_SCALED);
6825
6826 setValue(&I, Histogram);
6827 DAG.setRoot(Histogram);
6828}
6829
6830void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6831 unsigned Intrinsic) {
6832 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6833 "Tried lowering invalid vector extract last");
6834 SDLoc sdl = getCurSDLoc();
6835 const DataLayout &Layout = DAG.getDataLayout();
6836 SDValue Data = getValue(I.getOperand(0));
6837 SDValue Mask = getValue(I.getOperand(1));
6838
6839 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6840 EVT ResVT = TLI.getValueType(Layout, I.getType());
6841
6842 EVT ExtVT = TLI.getVectorIdxTy(Layout);
6843 SDValue Idx = DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, sdl, ExtVT, Mask);
6844 SDValue Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, sdl, ResVT, Data, Idx);
6845
6846 Value *Default = I.getOperand(2);
6848 SDValue PassThru = getValue(Default);
6849 EVT BoolVT = Mask.getValueType().getScalarType();
6850 SDValue AnyActive = DAG.getNode(ISD::VECREDUCE_OR, sdl, BoolVT, Mask);
6851 Result = DAG.getSelect(sdl, ResVT, AnyActive, Result, PassThru);
6852 }
6853
6854 setValue(&I, Result);
6855}
6856
6857/// Lower the call to the specified intrinsic function.
6858void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6859 unsigned Intrinsic) {
6860 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6861 SDLoc sdl = getCurSDLoc();
6862 DebugLoc dl = getCurDebugLoc();
6863 SDValue Res;
6864
6865 SDNodeFlags Flags;
6866 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6867 Flags.copyFMF(*FPOp);
6868
6869 switch (Intrinsic) {
6870 default:
6871 // By default, turn this into a target intrinsic node.
6872 visitTargetIntrinsic(I, Intrinsic);
6873 return;
6874 case Intrinsic::vscale: {
6875 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6876 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6877 return;
6878 }
6879 case Intrinsic::vastart: visitVAStart(I); return;
6880 case Intrinsic::vaend: visitVAEnd(I); return;
6881 case Intrinsic::vacopy: visitVACopy(I); return;
6882 case Intrinsic::returnaddress:
6883 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
6884 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6885 getValue(I.getArgOperand(0))));
6886 return;
6887 case Intrinsic::addressofreturnaddress:
6888 setValue(&I,
6889 DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
6890 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6891 return;
6892 case Intrinsic::sponentry:
6893 setValue(&I,
6894 DAG.getNode(ISD::SPONENTRY, sdl,
6895 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6896 return;
6897 case Intrinsic::frameaddress:
6898 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
6899 TLI.getFrameIndexTy(DAG.getDataLayout()),
6900 getValue(I.getArgOperand(0))));
6901 return;
6902 case Intrinsic::read_volatile_register:
6903 case Intrinsic::read_register: {
6904 Value *Reg = I.getArgOperand(0);
6905 SDValue Chain = getRoot();
6907 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6908 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6909 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6910 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6911 setValue(&I, Res);
6912 DAG.setRoot(Res.getValue(1));
6913 return;
6914 }
6915 case Intrinsic::write_register: {
6916 Value *Reg = I.getArgOperand(0);
6917 Value *RegValue = I.getArgOperand(1);
6918 SDValue Chain = getRoot();
6920 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6921 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6922 RegName, getValue(RegValue)));
6923 return;
6924 }
6925 case Intrinsic::write_volatile_register: {
6926 Value *Reg = I.getArgOperand(0);
6927 Value *RegValue = I.getArgOperand(1);
6928 SDValue Chain = getRoot();
6929 const MDNode *MD = cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata());
6930 SDValue RegName = DAG.getMDNode(MD);
6931 EVT VT = TLI.getValueType(DAG.getDataLayout(), RegValue->getType());
6932 SDValue WriteChain = DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other,
6933 Chain, RegName, getValue(RegValue));
6934 // FAKE_USE of the physical register marks it live after the WRITE_REGISTER,
6935 // preventing the backend from dead-eliminating the write. This is
6936 // preferred over READ_REGISTER, which would emit extra register copies
6937 // (e.g. fmov xN, dN for FP/SIMD registers).
6938 const MDString *RegStr = cast<MDString>(MD->getOperand(0));
6939 LLT Ty = VT.isSimple() ? getLLTForMVT(VT.getSimpleVT()) : LLT();
6940 const MachineFunction &MF = DAG.getMachineFunction();
6941 Register PhysReg =
6942 TLI.getRegisterByName(RegStr->getString().data(), Ty, MF);
6943 if (PhysReg.isValid()) {
6944 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6945 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(PhysReg);
6946 MVT RegVT = *TRI->legalclasstypes_begin(*RC);
6947 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other,
6948 {WriteChain, DAG.getRegister(PhysReg, RegVT)}));
6949 } else {
6950 DAG.setRoot(WriteChain);
6951 }
6952 return;
6953 }
6954 case Intrinsic::memcpy:
6955 case Intrinsic::memcpy_inline: {
6956 const auto &MCI = cast<MemCpyInst>(I);
6957 SDValue Dst = getValue(I.getArgOperand(0));
6958 SDValue Src = getValue(I.getArgOperand(1));
6959 SDValue Size = getValue(I.getArgOperand(2));
6960 assert((!MCI.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6961 "memcpy_inline needs constant size");
6962 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6963 Align DstAlign = MCI.getDestAlign().valueOrOne();
6964 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6965 bool isVol = MCI.isVolatile();
6966 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6967 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, DstAlign, SrcAlign,
6968 isVol, MCI.isForceInlined(), &I, std::nullopt,
6969 MachinePointerInfo(I.getArgOperand(0)),
6970 MachinePointerInfo(I.getArgOperand(1)),
6971 I.getAAMetadata(), BatchAA);
6972 updateDAGForMaybeTailCall(MC);
6973 return;
6974 }
6975 case Intrinsic::memset:
6976 case Intrinsic::memset_inline: {
6977 const auto &MSII = cast<MemSetInst>(I);
6978 SDValue Dst = getValue(I.getArgOperand(0));
6979 SDValue Value = getValue(I.getArgOperand(1));
6980 SDValue Size = getValue(I.getArgOperand(2));
6981 assert((!MSII.isForceInlined() || isa<ConstantSDNode>(Size)) &&
6982 "memset_inline needs constant size");
6983 // @llvm.memset defines 0 and 1 to both mean no alignment.
6984 Align DstAlign = MSII.getDestAlign().valueOrOne();
6985 bool isVol = MSII.isVolatile();
6986 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6987 SDValue MC = DAG.getMemset(
6988 Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(),
6989 &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6990 updateDAGForMaybeTailCall(MC);
6991 return;
6992 }
6993 case Intrinsic::memmove: {
6994 const auto &MMI = cast<MemMoveInst>(I);
6995 SDValue Op1 = getValue(I.getArgOperand(0));
6996 SDValue Op2 = getValue(I.getArgOperand(1));
6997 SDValue Op3 = getValue(I.getArgOperand(2));
6998 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6999 Align DstAlign = MMI.getDestAlign().valueOrOne();
7000 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
7001 bool isVol = MMI.isVolatile();
7002 SDValue Root = isVol ? getRoot() : getMemoryRoot();
7003 SDValue MM = DAG.getMemmove(
7004 Root, sdl, Op1, Op2, Op3, DstAlign, SrcAlign, isVol, &I,
7005 /* OverrideTailCall */ std::nullopt,
7006 MachinePointerInfo(I.getArgOperand(0)),
7007 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), BatchAA);
7008 updateDAGForMaybeTailCall(MM);
7009 return;
7010 }
7011 case Intrinsic::memcpy_element_unordered_atomic: {
7012 auto &MI = cast<AnyMemCpyInst>(I);
7013 SDValue Dst = getValue(MI.getRawDest());
7014 SDValue Src = getValue(MI.getRawSource());
7015 SDValue Length = getValue(MI.getLength());
7016
7017 Type *LengthTy = MI.getLength()->getType();
7018 unsigned ElemSz = MI.getElementSizeInBytes();
7019 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7020 SDValue MC =
7021 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
7022 isTC, MachinePointerInfo(MI.getRawDest()),
7023 MachinePointerInfo(MI.getRawSource()));
7024 updateDAGForMaybeTailCall(MC);
7025 return;
7026 }
7027 case Intrinsic::memmove_element_unordered_atomic: {
7028 auto &MI = cast<AnyMemMoveInst>(I);
7029 SDValue Dst = getValue(MI.getRawDest());
7030 SDValue Src = getValue(MI.getRawSource());
7031 SDValue Length = getValue(MI.getLength());
7032
7033 Type *LengthTy = MI.getLength()->getType();
7034 unsigned ElemSz = MI.getElementSizeInBytes();
7035 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7036 SDValue MC =
7037 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
7038 isTC, MachinePointerInfo(MI.getRawDest()),
7039 MachinePointerInfo(MI.getRawSource()));
7040 updateDAGForMaybeTailCall(MC);
7041 return;
7042 }
7043 case Intrinsic::memset_element_unordered_atomic: {
7044 auto &MI = cast<AnyMemSetInst>(I);
7045 SDValue Dst = getValue(MI.getRawDest());
7046 SDValue Val = getValue(MI.getValue());
7047 SDValue Length = getValue(MI.getLength());
7048
7049 Type *LengthTy = MI.getLength()->getType();
7050 unsigned ElemSz = MI.getElementSizeInBytes();
7051 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
7052 SDValue MC =
7053 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
7054 isTC, MachinePointerInfo(MI.getRawDest()));
7055 updateDAGForMaybeTailCall(MC);
7056 return;
7057 }
7058 case Intrinsic::call_preallocated_setup: {
7059 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
7060 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
7061 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
7062 getRoot(), SrcValue);
7063 setValue(&I, Res);
7064 DAG.setRoot(Res);
7065 return;
7066 }
7067 case Intrinsic::call_preallocated_arg: {
7068 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
7069 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
7070 SDValue Ops[3];
7071 Ops[0] = getRoot();
7072 Ops[1] = SrcValue;
7073 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7074 MVT::i32); // arg index
7075 SDValue Res = DAG.getNode(
7077 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
7078 setValue(&I, Res);
7079 DAG.setRoot(Res.getValue(1));
7080 return;
7081 }
7082
7083 case Intrinsic::eh_typeid_for: {
7084 // Find the type id for the given typeinfo.
7085 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
7086 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
7087 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
7088 setValue(&I, Res);
7089 return;
7090 }
7091
7092 case Intrinsic::eh_return_i32:
7093 case Intrinsic::eh_return_i64:
7094 DAG.getMachineFunction().setCallsEHReturn(true);
7095 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
7096 MVT::Other,
7098 getValue(I.getArgOperand(0)),
7099 getValue(I.getArgOperand(1))));
7100 return;
7101 case Intrinsic::eh_unwind_init:
7102 DAG.getMachineFunction().setCallsUnwindInit(true);
7103 return;
7104 case Intrinsic::eh_dwarf_cfa:
7105 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
7106 TLI.getPointerTy(DAG.getDataLayout()),
7107 getValue(I.getArgOperand(0))));
7108 return;
7109 case Intrinsic::eh_sjlj_callsite: {
7110 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
7111 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
7112
7113 FuncInfo.setCurrentCallSite(CI->getZExtValue());
7114 return;
7115 }
7116 case Intrinsic::eh_sjlj_functioncontext: {
7117 // Get and store the index of the function context.
7118 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7119 AllocaInst *FnCtx =
7120 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
7121 int FI = FuncInfo.StaticAllocaMap[FnCtx];
7123 return;
7124 }
7125 case Intrinsic::eh_sjlj_setjmp: {
7126 SDValue Ops[2];
7127 Ops[0] = getRoot();
7128 Ops[1] = getValue(I.getArgOperand(0));
7129 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
7130 DAG.getVTList(MVT::i32, MVT::Other), Ops);
7131 setValue(&I, Op.getValue(0));
7132 DAG.setRoot(Op.getValue(1));
7133 return;
7134 }
7135 case Intrinsic::eh_sjlj_longjmp:
7136 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
7137 getRoot(), getValue(I.getArgOperand(0))));
7138 return;
7139 case Intrinsic::eh_sjlj_setup_dispatch:
7140 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
7141 getRoot()));
7142 return;
7143 case Intrinsic::masked_gather:
7144 visitMaskedGather(I);
7145 return;
7146 case Intrinsic::masked_load:
7147 visitMaskedLoad(I);
7148 return;
7149 case Intrinsic::masked_scatter:
7150 visitMaskedScatter(I);
7151 return;
7152 case Intrinsic::masked_store:
7153 visitMaskedStore(I);
7154 return;
7155 case Intrinsic::masked_expandload:
7156 visitMaskedLoad(I, true /* IsExpanding */);
7157 return;
7158 case Intrinsic::masked_compressstore:
7159 visitMaskedStore(I, true /* IsCompressing */);
7160 return;
7161 case Intrinsic::speculative_load:
7162 visitSpeculativeLoad(I);
7163 return;
7164 case Intrinsic::powi:
7165 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
7166 getValue(I.getArgOperand(1)), DAG));
7167 return;
7168 case Intrinsic::log:
7169 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7170 return;
7171 case Intrinsic::log2:
7172 setValue(&I,
7173 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7174 return;
7175 case Intrinsic::log10:
7176 setValue(&I,
7177 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7178 return;
7179 case Intrinsic::exp:
7180 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7181 return;
7182 case Intrinsic::exp2:
7183 setValue(&I,
7184 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
7185 return;
7186 case Intrinsic::pow:
7187 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
7188 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
7189 return;
7190 case Intrinsic::sqrt:
7191 case Intrinsic::fabs:
7192 case Intrinsic::sin:
7193 case Intrinsic::cos:
7194 case Intrinsic::tan:
7195 case Intrinsic::asin:
7196 case Intrinsic::acos:
7197 case Intrinsic::atan:
7198 case Intrinsic::sinh:
7199 case Intrinsic::cosh:
7200 case Intrinsic::tanh:
7201 case Intrinsic::exp10:
7202 case Intrinsic::floor:
7203 case Intrinsic::ceil:
7204 case Intrinsic::trunc:
7205 case Intrinsic::rint:
7206 case Intrinsic::nearbyint:
7207 case Intrinsic::round:
7208 case Intrinsic::roundeven:
7209 case Intrinsic::canonicalize: {
7210 unsigned Opcode;
7211 // clang-format off
7212 switch (Intrinsic) {
7213 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7214 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
7215 case Intrinsic::fabs: Opcode = ISD::FABS; break;
7216 case Intrinsic::sin: Opcode = ISD::FSIN; break;
7217 case Intrinsic::cos: Opcode = ISD::FCOS; break;
7218 case Intrinsic::tan: Opcode = ISD::FTAN; break;
7219 case Intrinsic::asin: Opcode = ISD::FASIN; break;
7220 case Intrinsic::acos: Opcode = ISD::FACOS; break;
7221 case Intrinsic::atan: Opcode = ISD::FATAN; break;
7222 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
7223 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
7224 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
7225 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
7226 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
7227 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
7228 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
7229 case Intrinsic::rint: Opcode = ISD::FRINT; break;
7230 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
7231 case Intrinsic::round: Opcode = ISD::FROUND; break;
7232 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
7233 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
7234 }
7235 // clang-format on
7236
7237 setValue(&I, DAG.getNode(Opcode, sdl,
7238 getValue(I.getArgOperand(0)).getValueType(),
7239 getValue(I.getArgOperand(0)), Flags));
7240 return;
7241 }
7242 case Intrinsic::atan2:
7243 setValue(&I, DAG.getNode(ISD::FATAN2, sdl,
7244 getValue(I.getArgOperand(0)).getValueType(),
7245 getValue(I.getArgOperand(0)),
7246 getValue(I.getArgOperand(1)), Flags));
7247 return;
7248 case Intrinsic::lround:
7249 case Intrinsic::llround:
7250 case Intrinsic::lrint:
7251 case Intrinsic::llrint: {
7252 unsigned Opcode;
7253 // clang-format off
7254 switch (Intrinsic) {
7255 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7256 case Intrinsic::lround: Opcode = ISD::LROUND; break;
7257 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
7258 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
7259 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
7260 }
7261 // clang-format on
7262
7263 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7264 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
7265 getValue(I.getArgOperand(0))));
7266 return;
7267 }
7268 case Intrinsic::minnum:
7269 setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
7270 getValue(I.getArgOperand(0)).getValueType(),
7271 getValue(I.getArgOperand(0)),
7272 getValue(I.getArgOperand(1)), Flags));
7273 return;
7274 case Intrinsic::maxnum:
7275 setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
7276 getValue(I.getArgOperand(0)).getValueType(),
7277 getValue(I.getArgOperand(0)),
7278 getValue(I.getArgOperand(1)), Flags));
7279 return;
7280 case Intrinsic::minimum:
7281 setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
7282 getValue(I.getArgOperand(0)).getValueType(),
7283 getValue(I.getArgOperand(0)),
7284 getValue(I.getArgOperand(1)), Flags));
7285 return;
7286 case Intrinsic::maximum:
7287 setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
7288 getValue(I.getArgOperand(0)).getValueType(),
7289 getValue(I.getArgOperand(0)),
7290 getValue(I.getArgOperand(1)), Flags));
7291 return;
7292 case Intrinsic::minimumnum:
7293 setValue(&I, DAG.getNode(ISD::FMINIMUMNUM, sdl,
7294 getValue(I.getArgOperand(0)).getValueType(),
7295 getValue(I.getArgOperand(0)),
7296 getValue(I.getArgOperand(1)), Flags));
7297 return;
7298 case Intrinsic::maximumnum:
7299 setValue(&I, DAG.getNode(ISD::FMAXIMUMNUM, sdl,
7300 getValue(I.getArgOperand(0)).getValueType(),
7301 getValue(I.getArgOperand(0)),
7302 getValue(I.getArgOperand(1)), Flags));
7303 return;
7304 case Intrinsic::copysign:
7305 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
7306 getValue(I.getArgOperand(0)).getValueType(),
7307 getValue(I.getArgOperand(0)),
7308 getValue(I.getArgOperand(1)), Flags));
7309 return;
7310 case Intrinsic::ldexp:
7311 setValue(&I, DAG.getNode(ISD::FLDEXP, sdl,
7312 getValue(I.getArgOperand(0)).getValueType(),
7313 getValue(I.getArgOperand(0)),
7314 getValue(I.getArgOperand(1)), Flags));
7315 return;
7316 case Intrinsic::modf:
7317 case Intrinsic::sincos:
7318 case Intrinsic::sincospi:
7319 case Intrinsic::frexp: {
7320 unsigned Opcode;
7321 switch (Intrinsic) {
7322 default:
7323 llvm_unreachable("unexpected intrinsic");
7324 case Intrinsic::sincos:
7325 Opcode = ISD::FSINCOS;
7326 break;
7327 case Intrinsic::sincospi:
7328 Opcode = ISD::FSINCOSPI;
7329 break;
7330 case Intrinsic::modf:
7331 Opcode = ISD::FMODF;
7332 break;
7333 case Intrinsic::frexp:
7334 Opcode = ISD::FFREXP;
7335 break;
7336 }
7337 SmallVector<EVT, 2> ValueVTs;
7338 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
7339 SDVTList VTs = DAG.getVTList(ValueVTs);
7340 setValue(
7341 &I, DAG.getNode(Opcode, sdl, VTs, getValue(I.getArgOperand(0)), Flags));
7342 return;
7343 }
7344 case Intrinsic::arithmetic_fence: {
7345 setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
7346 getValue(I.getArgOperand(0)).getValueType(),
7347 getValue(I.getArgOperand(0)), Flags));
7348 return;
7349 }
7350 case Intrinsic::fma:
7351 setValue(&I, DAG.getNode(
7352 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
7353 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
7354 getValue(I.getArgOperand(2)), Flags));
7355 return;
7356#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7357 case Intrinsic::INTRINSIC:
7358#include "llvm/IR/ConstrainedOps.def"
7359 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
7360 return;
7361#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7362#include "llvm/IR/VPIntrinsics.def"
7363 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
7364 return;
7365 case Intrinsic::fptrunc_round: {
7366 // Get the last argument, the metadata and convert it to an integer in the
7367 // call
7368 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7369 std::optional<RoundingMode> RoundMode =
7370 convertStrToRoundingMode(cast<MDString>(MD)->getString());
7371
7372 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7373
7374 // Propagate fast-math-flags from IR to node(s).
7375 SDNodeFlags Flags;
7376 Flags.copyFMF(*cast<FPMathOperator>(&I));
7377 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
7378
7380 Result = DAG.getNode(
7381 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
7382 DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));
7383 setValue(&I, Result);
7384
7385 return;
7386 }
7387 case Intrinsic::fmuladd: {
7388 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7389 if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
7390 setValue(&I, DAG.getNode(ISD::FMA, sdl,
7391 getValue(I.getArgOperand(0)).getValueType(),
7392 getValue(I.getArgOperand(0)),
7393 getValue(I.getArgOperand(1)),
7394 getValue(I.getArgOperand(2)), Flags));
7395 } else if (TLI.isOperationLegalOrCustom(ISD::FMULADD, VT)) {
7396 // TODO: Support splitting the vector.
7397 setValue(&I, DAG.getNode(ISD::FMULADD, sdl,
7398 getValue(I.getArgOperand(0)).getValueType(),
7399 getValue(I.getArgOperand(0)),
7400 getValue(I.getArgOperand(1)),
7401 getValue(I.getArgOperand(2)), Flags));
7402 } else {
7403 // TODO: Intrinsic calls should have fast-math-flags.
7404 SDValue Mul = DAG.getNode(
7405 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
7406 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
7407 SDValue Add = DAG.getNode(ISD::FADD, sdl,
7408 getValue(I.getArgOperand(0)).getValueType(),
7409 Mul, getValue(I.getArgOperand(2)), Flags);
7410 setValue(&I, Add);
7411 }
7412 return;
7413 }
7414 case Intrinsic::fptosi_sat: {
7415 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7416 setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
7417 getValue(I.getArgOperand(0)),
7418 DAG.getValueType(VT.getScalarType())));
7419 return;
7420 }
7421 case Intrinsic::fptoui_sat: {
7422 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7423 setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
7424 getValue(I.getArgOperand(0)),
7425 DAG.getValueType(VT.getScalarType())));
7426 return;
7427 }
7428 case Intrinsic::convert_from_arbitrary_fp: {
7429 // Extract format metadata and convert to semantics enum.
7430 EVT DstVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7431 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7432 StringRef FormatStr = cast<MDString>(MD)->getString();
7433 const fltSemantics *SrcSem =
7435 if (!SrcSem) {
7436 DAG.getContext()->emitError(
7437 "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7438 "'");
7439 setValue(&I, DAG.getPOISON(DstVT));
7440 return;
7441 }
7443
7444 SDValue IntVal = getValue(I.getArgOperand(0));
7445
7446 // Emit ISD::CONVERT_FROM_ARBITRARY_FP node.
7447 SDValue SemConst =
7448 DAG.getTargetConstant(static_cast<int>(SemEnum), sdl, MVT::i32);
7449 setValue(&I, DAG.getNode(ISD::CONVERT_FROM_ARBITRARY_FP, sdl, DstVT, IntVal,
7450 SemConst));
7451 return;
7452 }
7453 case Intrinsic::convert_to_arbitrary_fp: {
7454 // Extract format metadata and convert to semantics enum.
7455 EVT DstVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7456 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7457 StringRef FormatStr = cast<MDString>(MD)->getString();
7458 const fltSemantics *DstSem =
7460 if (!DstSem) {
7461 DAG.getContext()->emitError(
7462 "convert_to_arbitrary_fp: not implemented format '" + FormatStr +
7463 "'");
7464 setValue(&I, DAG.getPOISON(DstVT));
7465 return;
7466 }
7468
7469 Metadata *RoundMD =
7470 cast<MetadataAsValue>(I.getArgOperand(2))->getMetadata();
7471 StringRef RoundStr = cast<MDString>(RoundMD)->getString();
7472 std::optional<RoundingMode> RoundMode = convertStrToRoundingMode(RoundStr);
7473 assert(RoundMode && *RoundMode != RoundingMode::Dynamic &&
7474 "Dynamic rounding mode should have been rejected by the verifier");
7475
7476 uint64_t Saturate =
7477 cast<ConstantInt>(I.getArgOperand(3))->getZExtValue() ? 1 : 0;
7478
7479 SDValue FloatVal = getValue(I.getArgOperand(0));
7480
7481 SDValue SemConst =
7482 DAG.getTargetConstant(static_cast<int>(SemEnum), sdl, MVT::i32);
7483 SDValue RoundConst =
7484 DAG.getTargetConstant(static_cast<int>(*RoundMode), sdl, MVT::i32);
7485 SDValue SatConst = DAG.getTargetConstant(Saturate, sdl, MVT::i32);
7486 setValue(&I, DAG.getNode(ISD::CONVERT_TO_ARBITRARY_FP, sdl, DstVT, FloatVal,
7487 SemConst, RoundConst, SatConst));
7488 return;
7489 }
7490 case Intrinsic::set_rounding:
7491 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
7492 {getRoot(), getValue(I.getArgOperand(0))});
7493 setValue(&I, Res);
7494 DAG.setRoot(Res.getValue(0));
7495 return;
7496 case Intrinsic::is_fpclass: {
7497 const DataLayout DLayout = DAG.getDataLayout();
7498 EVT DestVT = TLI.getValueType(DLayout, I.getType());
7499 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
7500 FPClassTest Test = static_cast<FPClassTest>(
7501 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
7502 MachineFunction &MF = DAG.getMachineFunction();
7503 const Function &F = MF.getFunction();
7504 SDValue Op = getValue(I.getArgOperand(0));
7505 SDNodeFlags Flags;
7506 Flags.setNoFPExcept(
7507 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7508 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7509 // expansion can use illegal types. Making expansion early allows
7510 // legalizing these types prior to selection.
7511 if (!TLI.isOperationLegal(ISD::IS_FPCLASS, ArgVT) &&
7513 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
7514 setValue(&I, Result);
7515 return;
7516 }
7517
7518 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
7519 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
7520 setValue(&I, V);
7521 return;
7522 }
7523 case Intrinsic::get_fpenv: {
7524 const DataLayout DLayout = DAG.getDataLayout();
7525 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
7526 Align TempAlign = DAG.getEVTAlign(EnvVT);
7527 SDValue Chain = getRoot();
7528 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7529 // and temporary storage in stack.
7531 Res = DAG.getNode(
7532 ISD::GET_FPENV, sdl,
7533 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7534 MVT::Other),
7535 Chain);
7536 } else {
7537 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7538 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7539 auto MPI =
7540 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
7541 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7543 TempAlign);
7544 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7545 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
7546 }
7547 setValue(&I, Res);
7548 DAG.setRoot(Res.getValue(1));
7549 return;
7550 }
7551 case Intrinsic::set_fpenv: {
7552 SDValue Env = getValue(I.getArgOperand(0));
7553 EVT EnvVT = Env.getValueType();
7554 Align TempAlign = DAG.getEVTAlign(EnvVT);
7555 SDValue Chain = getRoot();
7556 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7557 // environment from memory.
7559 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
7560 } else {
7561 // Allocate space in stack, copy environment bits into it and use this
7562 // memory in SET_FPENV_MEM.
7563 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7564 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7565 auto MPI =
7566 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
7567 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7569 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
7571 TempAlign);
7572 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7573 }
7574 DAG.setRoot(Chain);
7575 return;
7576 }
7577 case Intrinsic::reset_fpenv:
7578 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
7579 return;
7580 case Intrinsic::get_fpmode:
7581 Res = DAG.getNode(
7582 ISD::GET_FPMODE, sdl,
7583 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7584 MVT::Other),
7585 DAG.getRoot());
7586 setValue(&I, Res);
7587 DAG.setRoot(Res.getValue(1));
7588 return;
7589 case Intrinsic::set_fpmode:
7590 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
7591 getValue(I.getArgOperand(0)));
7592 DAG.setRoot(Res);
7593 return;
7594 case Intrinsic::reset_fpmode: {
7595 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
7596 DAG.setRoot(Res);
7597 return;
7598 }
7599 case Intrinsic::pcmarker: {
7600 SDValue Tmp = getValue(I.getArgOperand(0));
7601 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
7602 return;
7603 }
7604 case Intrinsic::readcyclecounter: {
7605 SDValue Op = getRoot();
7606 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
7607 DAG.getVTList(MVT::i64, MVT::Other), Op);
7608 setValue(&I, Res);
7609 DAG.setRoot(Res.getValue(1));
7610 return;
7611 }
7612 case Intrinsic::readsteadycounter: {
7613 SDValue Op = getRoot();
7614 Res = DAG.getNode(ISD::READSTEADYCOUNTER, sdl,
7615 DAG.getVTList(MVT::i64, MVT::Other), Op);
7616 setValue(&I, Res);
7617 DAG.setRoot(Res.getValue(1));
7618 return;
7619 }
7620 case Intrinsic::bitreverse:
7621 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
7622 getValue(I.getArgOperand(0)).getValueType(),
7623 getValue(I.getArgOperand(0))));
7624 return;
7625 case Intrinsic::bswap:
7626 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
7627 getValue(I.getArgOperand(0)).getValueType(),
7628 getValue(I.getArgOperand(0))));
7629 return;
7630 case Intrinsic::cttz: {
7631 SDValue Arg = getValue(I.getArgOperand(0));
7632 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7633 EVT Ty = Arg.getValueType();
7634 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_POISON,
7635 sdl, Ty, Arg));
7636 return;
7637 }
7638 case Intrinsic::ctlz: {
7639 SDValue Arg = getValue(I.getArgOperand(0));
7640 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7641 EVT Ty = Arg.getValueType();
7642 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_POISON,
7643 sdl, Ty, Arg));
7644 return;
7645 }
7646 case Intrinsic::ctpop: {
7647 SDValue Arg = getValue(I.getArgOperand(0));
7648 EVT Ty = Arg.getValueType();
7649 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7650 return;
7651 }
7652 case Intrinsic::fshl:
7653 case Intrinsic::fshr: {
7654 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7655 SDValue X = getValue(I.getArgOperand(0));
7656 SDValue Y = getValue(I.getArgOperand(1));
7657 SDValue Z = getValue(I.getArgOperand(2));
7658 EVT VT = X.getValueType();
7659
7660 if (X == Y) {
7661 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7662 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7663 } else {
7664 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7665 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7666 }
7667 return;
7668 }
7669 case Intrinsic::clmul: {
7670 SDValue X = getValue(I.getArgOperand(0));
7671 SDValue Y = getValue(I.getArgOperand(1));
7672 setValue(&I, DAG.getNode(ISD::CLMUL, sdl, X.getValueType(), X, Y));
7673 return;
7674 }
7675 case Intrinsic::pext: {
7676 SDValue X = getValue(I.getArgOperand(0));
7677 SDValue Y = getValue(I.getArgOperand(1));
7678 setValue(&I, DAG.getNode(ISD::PEXT, sdl, X.getValueType(), X, Y));
7679 return;
7680 }
7681 case Intrinsic::pdep: {
7682 SDValue X = getValue(I.getArgOperand(0));
7683 SDValue Y = getValue(I.getArgOperand(1));
7684 setValue(&I, DAG.getNode(ISD::PDEP, sdl, X.getValueType(), X, Y));
7685 return;
7686 }
7687 case Intrinsic::smulh:
7688 case Intrinsic::umulh: {
7689 auto Opc = Intrinsic == Intrinsic::smulh ? ISD::MULHS : ISD::MULHU;
7690 SDValue X = getValue(I.getArgOperand(0));
7691 SDValue Y = getValue(I.getArgOperand(1));
7692 setValue(&I, DAG.getNode(Opc, sdl, X.getValueType(), X, Y));
7693 return;
7694 }
7695 case Intrinsic::sadd_sat: {
7696 SDValue Op1 = getValue(I.getArgOperand(0));
7697 SDValue Op2 = getValue(I.getArgOperand(1));
7698 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7699 return;
7700 }
7701 case Intrinsic::uadd_sat: {
7702 SDValue Op1 = getValue(I.getArgOperand(0));
7703 SDValue Op2 = getValue(I.getArgOperand(1));
7704 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7705 return;
7706 }
7707 case Intrinsic::ssub_sat: {
7708 SDValue Op1 = getValue(I.getArgOperand(0));
7709 SDValue Op2 = getValue(I.getArgOperand(1));
7710 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7711 return;
7712 }
7713 case Intrinsic::usub_sat: {
7714 SDValue Op1 = getValue(I.getArgOperand(0));
7715 SDValue Op2 = getValue(I.getArgOperand(1));
7716 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7717 return;
7718 }
7719 case Intrinsic::sshl_sat:
7720 case Intrinsic::ushl_sat: {
7721 SDValue Op1 = getValue(I.getArgOperand(0));
7722 SDValue Op2 = getValue(I.getArgOperand(1));
7723
7724 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
7725 Op1.getValueType(), DAG.getDataLayout());
7726
7727 // Coerce the shift amount to the right type if we can. This exposes the
7728 // truncate or zext to optimization early.
7729 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
7730 assert(ShiftTy.getSizeInBits() >=
7732 "Unexpected shift type");
7733 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
7734 }
7735
7736 unsigned Opc =
7737 Intrinsic == Intrinsic::sshl_sat ? ISD::SSHLSAT : ISD::USHLSAT;
7738 setValue(&I, DAG.getNode(Opc, sdl, Op1.getValueType(), Op1, Op2));
7739 return;
7740 }
7741 case Intrinsic::smul_fix:
7742 case Intrinsic::umul_fix:
7743 case Intrinsic::smul_fix_sat:
7744 case Intrinsic::umul_fix_sat: {
7745 SDValue Op1 = getValue(I.getArgOperand(0));
7746 SDValue Op2 = getValue(I.getArgOperand(1));
7747 SDValue Op3 = getValue(I.getArgOperand(2));
7748 setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
7749 Op1.getValueType(), Op1, Op2, Op3));
7750 return;
7751 }
7752 case Intrinsic::sdiv_fix:
7753 case Intrinsic::udiv_fix:
7754 case Intrinsic::sdiv_fix_sat:
7755 case Intrinsic::udiv_fix_sat: {
7756 SDValue Op1 = getValue(I.getArgOperand(0));
7757 SDValue Op2 = getValue(I.getArgOperand(1));
7758 SDValue Op3 = getValue(I.getArgOperand(2));
7760 Op1, Op2, Op3, DAG, TLI));
7761 return;
7762 }
7763 case Intrinsic::smax: {
7764 SDValue Op1 = getValue(I.getArgOperand(0));
7765 SDValue Op2 = getValue(I.getArgOperand(1));
7766 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7767 return;
7768 }
7769 case Intrinsic::smin: {
7770 SDValue Op1 = getValue(I.getArgOperand(0));
7771 SDValue Op2 = getValue(I.getArgOperand(1));
7772 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7773 return;
7774 }
7775 case Intrinsic::umax: {
7776 SDValue Op1 = getValue(I.getArgOperand(0));
7777 SDValue Op2 = getValue(I.getArgOperand(1));
7778 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7779 return;
7780 }
7781 case Intrinsic::umin: {
7782 SDValue Op1 = getValue(I.getArgOperand(0));
7783 SDValue Op2 = getValue(I.getArgOperand(1));
7784 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7785 return;
7786 }
7787 case Intrinsic::abs: {
7788 SDValue Op1 = getValue(I.getArgOperand(0));
7789 bool IntMinIsPoison = cast<ConstantInt>(I.getArgOperand(1))->isOne();
7790 unsigned Opc = IntMinIsPoison ? ISD::ABS_MIN_POISON : ISD::ABS;
7791 setValue(&I, DAG.getNode(Opc, sdl, Op1.getValueType(), Op1));
7792 return;
7793 }
7794 case Intrinsic::scmp: {
7795 SDValue Op1 = getValue(I.getArgOperand(0));
7796 SDValue Op2 = getValue(I.getArgOperand(1));
7797 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7798 setValue(&I, DAG.getNode(ISD::SCMP, sdl, DestVT, Op1, Op2));
7799 break;
7800 }
7801 case Intrinsic::ucmp: {
7802 SDValue Op1 = getValue(I.getArgOperand(0));
7803 SDValue Op2 = getValue(I.getArgOperand(1));
7804 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7805 setValue(&I, DAG.getNode(ISD::UCMP, sdl, DestVT, Op1, Op2));
7806 break;
7807 }
7808 case Intrinsic::stackaddress:
7809 case Intrinsic::stacksave: {
7810 unsigned SDOpcode = Intrinsic == Intrinsic::stackaddress ? ISD::STACKADDRESS
7812 SDValue Op = getRoot();
7813 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7814 Res = DAG.getNode(SDOpcode, sdl, DAG.getVTList(VT, MVT::Other), Op);
7815 setValue(&I, Res);
7816 DAG.setRoot(Res.getValue(1));
7817 return;
7818 }
7819 case Intrinsic::stackrestore:
7820 Res = getValue(I.getArgOperand(0));
7821 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7822 return;
7823 case Intrinsic::get_dynamic_area_offset: {
7824 SDValue Op = getRoot();
7825 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7826 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
7827 Op);
7828 DAG.setRoot(Op);
7829 setValue(&I, Res);
7830 return;
7831 }
7832 case Intrinsic::stackguard: {
7833 MachineFunction &MF = DAG.getMachineFunction();
7834 const Module &M = *MF.getFunction().getParent();
7835 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7836 SDValue Chain = getRoot();
7837 if (TLI.useLoadStackGuardNode(M)) {
7838 Res = getLoadStackGuard(DAG, sdl, Chain);
7839 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7840 } else {
7841 const Value *Global = TLI.getSDagStackGuard(M, DAG.getLibcalls());
7842 if (!Global) {
7843 LLVMContext &Ctx = *DAG.getContext();
7844 Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));
7845 setValue(&I, DAG.getPOISON(PtrTy));
7846 return;
7847 }
7848
7849 Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
7850 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7851 MachinePointerInfo(Global, 0), Align,
7853 }
7854 // Mix the cookie with FP if enabled. Skip if using LOAD_STACK_GUARD
7855 // with post-RA mixing (AArch64 MSVCRT), as the mixing will be done during
7856 // post-RA expansion of LOAD_STACK_GUARD.
7858 Res = TLI.emitStackGuardMixFP(DAG, Res, sdl);
7859 DAG.setRoot(Chain);
7860 setValue(&I, Res);
7861 return;
7862 }
7863 case Intrinsic::stackprotector: {
7864 // Emit code into the DAG to store the stack guard onto the stack.
7865 MachineFunction &MF = DAG.getMachineFunction();
7866 MachineFrameInfo &MFI = MF.getFrameInfo();
7867 const Module &M = *MF.getFunction().getParent();
7868 SDValue Src, Chain = getRoot();
7869
7871 Src = getLoadStackGuard(DAG, sdl, Chain);
7872 else
7873 Src = getValue(I.getArgOperand(0)); // The guard's value.
7874
7875 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7876
7877 int FI = FuncInfo.StaticAllocaMap[Slot];
7878 MFI.setStackProtectorIndex(FI);
7879 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7880
7881 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7882
7883 // Store the stack protector onto the stack.
7884 Res = DAG.getStore(
7885 Chain, sdl, Src, FIN,
7886 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
7887 MaybeAlign(), MachineMemOperand::MOVolatile);
7888 setValue(&I, Res);
7889 DAG.setRoot(Res);
7890 return;
7891 }
7892 case Intrinsic::objectsize:
7893 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7894
7895 case Intrinsic::is_constant:
7896 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7897
7898 case Intrinsic::annotation:
7899 case Intrinsic::ptr_annotation:
7900 case Intrinsic::launder_invariant_group:
7901 // Drop the intrinsic, but forward the value
7902 setValue(&I, getValue(I.getOperand(0)));
7903 return;
7904
7905 case Intrinsic::type_test:
7906 case Intrinsic::public_type_test:
7907 case Intrinsic::type_checked_load:
7908 case Intrinsic::type_checked_load_relative: {
7909 // These intrinsics are expected to be lowered by the LowerTypeTests pass
7910 // before code generation. Surviving until here usually indicates a
7911 // misconfiguration, for instance when devirtualization is enabled but LTO
7912 // does not actually run.
7913 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
7914 *I.getFunction(),
7915 Intrinsic::getBaseName(Intrinsic) +
7916 " intrinsic must be lowered by the LowerTypeTests pass "
7917 "before code generation",
7918 sdl.getDebugLoc()));
7919
7920 // Lower the result to poison so that compilation can continue and collect
7921 // any further diagnostics.
7922 setValueToPoison(&I, sdl);
7923 return;
7924 }
7925
7926 case Intrinsic::assume:
7927 case Intrinsic::experimental_noalias_scope_decl:
7928 case Intrinsic::var_annotation:
7929 case Intrinsic::sideeffect:
7930 // Discard annotate attributes, noalias scope declarations, assumptions, and
7931 // artificial side-effects.
7932 return;
7933
7934 case Intrinsic::codeview_annotation: {
7935 // Emit a label associated with this metadata.
7936 MachineFunction &MF = DAG.getMachineFunction();
7937 MCSymbol *Label = MF.getContext().createTempSymbol("annotation", true);
7938 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7939 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7940 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7941 DAG.setRoot(Res);
7942 return;
7943 }
7944
7945 case Intrinsic::init_trampoline: {
7946 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7947
7948 SDValue Ops[6];
7949 Ops[0] = getRoot();
7950 Ops[1] = getValue(I.getArgOperand(0));
7951 Ops[2] = getValue(I.getArgOperand(1));
7952 Ops[3] = getValue(I.getArgOperand(2));
7953 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7954 Ops[5] = DAG.getSrcValue(F);
7955
7956 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7957
7958 DAG.setRoot(Res);
7959 return;
7960 }
7961 case Intrinsic::adjust_trampoline:
7962 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
7963 TLI.getPointerTy(DAG.getDataLayout()),
7964 getValue(I.getArgOperand(0))));
7965 return;
7966 case Intrinsic::gcroot: {
7967 assert(DAG.getMachineFunction().getFunction().hasGC() &&
7968 "only valid in functions with gc specified, enforced by Verifier");
7969 assert(GFI && "implied by previous");
7970 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7971 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7972
7973 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7974 GFI->addStackRoot(FI->getIndex(), TypeMap);
7975 return;
7976 }
7977 case Intrinsic::gcread:
7978 case Intrinsic::gcwrite:
7979 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7980 case Intrinsic::get_rounding:
7981 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7982 setValue(&I, Res);
7983 DAG.setRoot(Res.getValue(1));
7984 return;
7985
7986 case Intrinsic::expect:
7987 case Intrinsic::expect_with_probability:
7988 // Just replace __builtin_expect(exp, c) and
7989 // __builtin_expect_with_probability(exp, c, p) with EXP.
7990 setValue(&I, getValue(I.getArgOperand(0)));
7991 return;
7992
7993 case Intrinsic::ubsantrap:
7994 case Intrinsic::debugtrap:
7995 case Intrinsic::trap: {
7996 StringRef TrapFuncName =
7997 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
7998 if (TrapFuncName.empty()) {
7999 switch (Intrinsic) {
8000 case Intrinsic::trap:
8001 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
8002 break;
8003 case Intrinsic::debugtrap:
8004 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
8005 break;
8006 case Intrinsic::ubsantrap:
8007 DAG.setRoot(DAG.getNode(
8008 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
8009 DAG.getTargetConstant(
8010 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
8011 MVT::i32)));
8012 break;
8013 default: llvm_unreachable("unknown trap intrinsic");
8014 }
8015 DAG.addNoMergeSiteInfo(DAG.getRoot().getNode(),
8016 I.hasFnAttr(Attribute::NoMerge));
8017 return;
8018 }
8020 if (Intrinsic == Intrinsic::ubsantrap) {
8021 Value *Arg = I.getArgOperand(0);
8022 Args.emplace_back(Arg, getValue(Arg),
8023 LibInfo->getExtAttrForI8Param(/*Signed=*/false));
8024 }
8025
8026 TargetLowering::CallLoweringInfo CLI(DAG);
8027 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
8028 CallingConv::C, I.getType(),
8029 DAG.getExternalSymbol(TrapFuncName.data(),
8030 TLI.getPointerTy(DAG.getDataLayout())),
8031 std::move(Args));
8032 CLI.NoMerge = I.hasFnAttr(Attribute::NoMerge);
8033 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8034 DAG.setRoot(Result.second);
8035 return;
8036 }
8037
8038 case Intrinsic::allow_runtime_check:
8039 case Intrinsic::allow_ubsan_check:
8040 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
8041 return;
8042
8043 case Intrinsic::uadd_with_overflow:
8044 case Intrinsic::sadd_with_overflow:
8045 case Intrinsic::usub_with_overflow:
8046 case Intrinsic::ssub_with_overflow:
8047 case Intrinsic::umul_with_overflow:
8048 case Intrinsic::smul_with_overflow: {
8050 switch (Intrinsic) {
8051 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8052 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
8053 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
8054 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
8055 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
8056 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
8057 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
8058 }
8059 SDValue Op1 = getValue(I.getArgOperand(0));
8060 SDValue Op2 = getValue(I.getArgOperand(1));
8061
8062 EVT ResultVT = Op1.getValueType();
8063 EVT OverflowVT = ResultVT.changeElementType(*Context, MVT::i1);
8064
8065 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
8066 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
8067 return;
8068 }
8069 case Intrinsic::prefetch: {
8070 SDValue Ops[5];
8071 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
8073 Ops[0] = DAG.getRoot();
8074 Ops[1] = getValue(I.getArgOperand(0));
8075 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
8076 MVT::i32);
8077 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
8078 MVT::i32);
8079 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
8080 MVT::i32);
8081 SDValue Result = DAG.getMemIntrinsicNode(
8082 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
8083 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
8084 /* align */ std::nullopt, Flags);
8085
8086 // Chain the prefetch in parallel with any pending loads, to stay out of
8087 // the way of later optimizations.
8088 PendingLoads.push_back(Result);
8089 Result = getRoot();
8090 DAG.setRoot(Result);
8091 return;
8092 }
8093 case Intrinsic::lifetime_start:
8094 case Intrinsic::lifetime_end: {
8095 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
8096 // Stack coloring is not enabled in O0, discard region information.
8097 if (TM.getOptLevel() == CodeGenOptLevel::None)
8098 return;
8099
8100 const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(I.getArgOperand(0));
8101 if (!LifetimeObject)
8102 return;
8103
8104 // First check that the Alloca is static, otherwise it won't have a
8105 // valid frame index.
8106 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
8107 if (SI == FuncInfo.StaticAllocaMap.end())
8108 return;
8109
8110 const int FrameIndex = SI->second;
8111 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex);
8112 DAG.setRoot(Res);
8113 return;
8114 }
8115 case Intrinsic::pseudoprobe: {
8116 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
8117 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
8118 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
8119 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
8120 DAG.setRoot(Res);
8121 return;
8122 }
8123 case Intrinsic::invariant_start:
8124 // Discard region information.
8125 setValue(&I,
8126 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
8127 return;
8128 case Intrinsic::invariant_end:
8129 // Discard region information.
8130 return;
8131 case Intrinsic::clear_cache: {
8132 SDValue InputChain = DAG.getRoot();
8133 SDValue StartVal = getValue(I.getArgOperand(0));
8134 SDValue EndVal = getValue(I.getArgOperand(1));
8135 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),
8136 {InputChain, StartVal, EndVal});
8137 setValue(&I, Res);
8138 DAG.setRoot(Res);
8139 return;
8140 }
8141 case Intrinsic::donothing:
8142 case Intrinsic::seh_try_begin:
8143 case Intrinsic::seh_scope_begin:
8144 case Intrinsic::seh_try_end:
8145 case Intrinsic::seh_scope_end:
8146 // ignore
8147 return;
8148 case Intrinsic::experimental_stackmap:
8149 visitStackmap(I);
8150 return;
8151 case Intrinsic::experimental_patchpoint_void:
8152 case Intrinsic::experimental_patchpoint:
8153 visitPatchpoint(I);
8154 return;
8155 case Intrinsic::experimental_gc_statepoint:
8157 return;
8158 case Intrinsic::experimental_gc_result:
8159 visitGCResult(cast<GCResultInst>(I));
8160 return;
8161 case Intrinsic::experimental_gc_relocate:
8162 visitGCRelocate(cast<GCRelocateInst>(I));
8163 return;
8164 case Intrinsic::instrprof_cover:
8165 llvm_unreachable("instrprof failed to lower a cover");
8166 case Intrinsic::instrprof_increment:
8167 llvm_unreachable("instrprof failed to lower an increment");
8168 case Intrinsic::instrprof_timestamp:
8169 llvm_unreachable("instrprof failed to lower a timestamp");
8170 case Intrinsic::instrprof_value_profile:
8171 llvm_unreachable("instrprof failed to lower a value profiling call");
8172 case Intrinsic::instrprof_mcdc_parameters:
8173 llvm_unreachable("instrprof failed to lower mcdc parameters");
8174 case Intrinsic::instrprof_mcdc_tvbitmap_update:
8175 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
8176 case Intrinsic::localescape: {
8177 MachineFunction &MF = DAG.getMachineFunction();
8178 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
8179
8180 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
8181 // is the same on all targets.
8182 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
8183 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
8184 if (isa<ConstantPointerNull>(Arg))
8185 continue; // Skip null pointers. They represent a hole in index space.
8186 AllocaInst *Slot = cast<AllocaInst>(Arg);
8187 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
8188 "can only escape static allocas");
8189 int FI = FuncInfo.StaticAllocaMap[Slot];
8190 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8192 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
8193 TII->get(TargetOpcode::LOCAL_ESCAPE))
8194 .addSym(FrameAllocSym)
8195 .addFrameIndex(FI);
8196 }
8197
8198 return;
8199 }
8200
8201 case Intrinsic::localrecover: {
8202 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
8203 MachineFunction &MF = DAG.getMachineFunction();
8204
8205 // Get the symbol that defines the frame offset.
8206 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
8207 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
8208 unsigned IdxVal =
8209 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
8210 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
8212
8213 Value *FP = I.getArgOperand(1);
8214 SDValue FPVal = getValue(FP);
8215 EVT PtrVT = FPVal.getValueType();
8216
8217 // Create a MCSymbol for the label to avoid any target lowering
8218 // that would make this PC relative.
8219 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
8220 SDValue OffsetVal =
8221 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
8222
8223 // Add the offset to the FP.
8224 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
8225 setValue(&I, Add);
8226
8227 return;
8228 }
8229
8230 case Intrinsic::fake_use: {
8231 Value *V = I.getArgOperand(0);
8232 SDValue Ops[2];
8233 // For Values not declared or previously used in this basic block, the
8234 // NodeMap will not have an entry, and `getValue` will assert if V has no
8235 // valid register value.
8236 auto FakeUseValue = [&]() -> SDValue {
8237 SDValue &N = NodeMap[V];
8238 if (N.getNode())
8239 return N;
8240
8241 // If there's a virtual register allocated and initialized for this
8242 // value, use it.
8243 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
8244 return copyFromReg;
8245 // FIXME: Do we want to preserve constants? It seems pointless.
8246 if (isa<Constant>(V))
8247 return getValue(V);
8248 return SDValue();
8249 }();
8250 if (!FakeUseValue || FakeUseValue.isUndef())
8251 return;
8252 Ops[0] = getRoot();
8253 Ops[1] = FakeUseValue;
8254 // Also, do not translate a fake use with an undef operand, or any other
8255 // empty SDValues.
8256 if (!Ops[1] || Ops[1].isUndef())
8257 return;
8258 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other, Ops));
8259 return;
8260 }
8261
8262 case Intrinsic::reloc_none: {
8263 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
8264 StringRef SymbolName = cast<MDString>(MD)->getString();
8265 SDValue Ops[2] = {
8266 getRoot(),
8267 DAG.getTargetExternalSymbol(
8268 SymbolName.data(), TLI.getProgramPointerTy(DAG.getDataLayout()))};
8269 DAG.setRoot(DAG.getNode(ISD::RELOC_NONE, sdl, MVT::Other, Ops));
8270 return;
8271 }
8272
8273 case Intrinsic::cond_loop: {
8274 SDValue InputChain = DAG.getRoot();
8275 SDValue P = getValue(I.getArgOperand(0));
8276 Res = DAG.getNode(ISD::COND_LOOP, sdl, DAG.getVTList(MVT::Other),
8277 {InputChain, P});
8278 setValue(&I, Res);
8279 DAG.setRoot(Res);
8280 return;
8281 }
8282
8283 case Intrinsic::eh_exceptionpointer:
8284 case Intrinsic::eh_exceptioncode: {
8285 // Get the exception pointer vreg, copy from it, and resize it to fit.
8286 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
8287 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
8288 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
8289 Register VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
8290 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
8291 if (Intrinsic == Intrinsic::eh_exceptioncode)
8292 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
8293 setValue(&I, N);
8294 return;
8295 }
8296 case Intrinsic::xray_customevent: {
8297 // Here we want to make sure that the intrinsic behaves as if it has a
8298 // specific calling convention.
8299 const auto &Triple = DAG.getTarget().getTargetTriple();
8300 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64 &&
8301 Triple.getArch() != Triple::hexagon)
8302 return;
8303
8305
8306 // We want to say that we always want the arguments in registers.
8307 SDValue LogEntryVal = getValue(I.getArgOperand(0));
8308 SDValue StrSizeVal = getValue(I.getArgOperand(1));
8309 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8310 SDValue Chain = getRoot();
8311 Ops.push_back(LogEntryVal);
8312 Ops.push_back(StrSizeVal);
8313 Ops.push_back(Chain);
8314
8315 // We need to enforce the calling convention for the callsite, so that
8316 // argument ordering is enforced correctly, and that register allocation can
8317 // see that some registers may be assumed clobbered and have to preserve
8318 // them across calls to the intrinsic.
8319 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
8320 sdl, NodeTys, Ops);
8321 SDValue patchableNode = SDValue(MN, 0);
8322 DAG.setRoot(patchableNode);
8323 setValue(&I, patchableNode);
8324 return;
8325 }
8326 case Intrinsic::xray_typedevent: {
8327 // Here we want to make sure that the intrinsic behaves as if it has a
8328 // specific calling convention.
8329 const auto &Triple = DAG.getTarget().getTargetTriple();
8330 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64 &&
8331 Triple.getArch() != Triple::hexagon)
8332 return;
8333
8335
8336 // We want to say that we always want the arguments in registers.
8337 // It's unclear to me how manipulating the selection DAG here forces callers
8338 // to provide arguments in registers instead of on the stack.
8339 SDValue LogTypeId = getValue(I.getArgOperand(0));
8340 SDValue LogEntryVal = getValue(I.getArgOperand(1));
8341 SDValue StrSizeVal = getValue(I.getArgOperand(2));
8342 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8343 SDValue Chain = getRoot();
8344 Ops.push_back(LogTypeId);
8345 Ops.push_back(LogEntryVal);
8346 Ops.push_back(StrSizeVal);
8347 Ops.push_back(Chain);
8348
8349 // We need to enforce the calling convention for the callsite, so that
8350 // argument ordering is enforced correctly, and that register allocation can
8351 // see that some registers may be assumed clobbered and have to preserve
8352 // them across calls to the intrinsic.
8353 MachineSDNode *MN = DAG.getMachineNode(
8354 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
8355 SDValue patchableNode = SDValue(MN, 0);
8356 DAG.setRoot(patchableNode);
8357 setValue(&I, patchableNode);
8358 return;
8359 }
8360 case Intrinsic::experimental_deoptimize:
8362 return;
8363 case Intrinsic::stepvector:
8364 visitStepVector(I);
8365 return;
8366 case Intrinsic::vector_reduce_fadd:
8367 case Intrinsic::vector_reduce_fmul:
8368 case Intrinsic::vector_reduce_add:
8369 case Intrinsic::vector_reduce_mul:
8370 case Intrinsic::vector_reduce_and:
8371 case Intrinsic::vector_reduce_or:
8372 case Intrinsic::vector_reduce_xor:
8373 case Intrinsic::vector_reduce_smax:
8374 case Intrinsic::vector_reduce_smin:
8375 case Intrinsic::vector_reduce_umax:
8376 case Intrinsic::vector_reduce_umin:
8377 case Intrinsic::vector_reduce_fmax:
8378 case Intrinsic::vector_reduce_fmin:
8379 case Intrinsic::vector_reduce_fmaximum:
8380 case Intrinsic::vector_reduce_fminimum:
8381 case Intrinsic::vector_reduce_fmaximumnum:
8382 case Intrinsic::vector_reduce_fminimumnum:
8383 visitVectorReduce(I, Intrinsic);
8384 return;
8385
8386 case Intrinsic::icall_branch_funnel: {
8388 Ops.push_back(getValue(I.getArgOperand(0)));
8389
8390 int64_t Offset;
8392 I.getArgOperand(1), Offset, DAG.getDataLayout()));
8393 if (!Base)
8395 "llvm.icall.branch.funnel operand must be a GlobalValue");
8396 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
8397
8398 struct BranchFunnelTarget {
8399 int64_t Offset;
8401 };
8403
8404 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
8406 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
8407 if (ElemBase != Base)
8408 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
8409 "to the same GlobalValue");
8410
8411 SDValue Val = getValue(I.getArgOperand(Op + 1));
8412 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
8413 if (!GA)
8415 "llvm.icall.branch.funnel operand must be a GlobalValue");
8416 Targets.push_back({Offset, DAG.getTargetGlobalAddress(
8417 GA->getGlobal(), sdl, Val.getValueType(),
8418 GA->getOffset())});
8419 }
8420 llvm::sort(Targets,
8421 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
8422 return T1.Offset < T2.Offset;
8423 });
8424
8425 for (auto &T : Targets) {
8426 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
8427 Ops.push_back(T.Target);
8428 }
8429
8430 Ops.push_back(DAG.getRoot()); // Chain
8431 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
8432 MVT::Other, Ops),
8433 0);
8434 DAG.setRoot(N);
8435 setValue(&I, N);
8436 HasTailCall = true;
8437 return;
8438 }
8439
8440 case Intrinsic::wasm_landingpad_index:
8441 // Information this intrinsic contained has been transferred to
8442 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
8443 // delete it now.
8444 return;
8445
8446 case Intrinsic::aarch64_settag:
8447 case Intrinsic::aarch64_settag_zero: {
8448 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
8449 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
8451 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
8452 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
8453 ZeroMemory);
8454 DAG.setRoot(Val);
8455 setValue(&I, Val);
8456 return;
8457 }
8458 case Intrinsic::amdgcn_cs_chain: {
8459 // At this point we don't care if it's amdgpu_cs_chain or
8460 // amdgpu_cs_chain_preserve.
8462
8463 Type *RetTy = I.getType();
8464 assert(RetTy->isVoidTy() && "Should not return");
8465
8466 SDValue Callee = getValue(I.getOperand(0));
8467
8468 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
8469 // We'll also tack the value of the EXEC mask at the end.
8471 Args.reserve(3);
8472
8473 for (unsigned Idx : {2, 3, 1}) {
8474 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
8475 I.getOperand(Idx)->getType());
8476 Arg.setAttributes(&I, Idx);
8477 Args.push_back(Arg);
8478 }
8479
8480 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
8481 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
8482 Args[2].IsInReg = true; // EXEC should be inreg
8483
8484 // Forward the flags and any additional arguments.
8485 for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
8486 TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
8487 I.getOperand(Idx)->getType());
8488 Arg.setAttributes(&I, Idx);
8489 Args.push_back(Arg);
8490 }
8491
8492 TargetLowering::CallLoweringInfo CLI(DAG);
8493 CLI.setDebugLoc(getCurSDLoc())
8494 .setChain(getRoot())
8495 .setCallee(CC, RetTy, Callee, std::move(Args))
8496 .setNoReturn(true)
8497 .setTailCall(true)
8498 .setConvergent(I.isConvergent());
8499 CLI.CB = &I;
8500 std::pair<SDValue, SDValue> Result =
8501 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
8502 (void)Result;
8503 assert(!Result.first.getNode() && !Result.second.getNode() &&
8504 "Should've lowered as tail call");
8505
8506 HasTailCall = true;
8507 return;
8508 }
8509 case Intrinsic::amdgcn_call_whole_wave: {
8511 bool isTailCall = I.isTailCall();
8512
8513 // The first argument is the callee. Skip it when assembling the call args.
8514 for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
8515 TargetLowering::ArgListEntry Arg(getValue(I.getArgOperand(Idx)),
8516 I.getArgOperand(Idx)->getType());
8517 Arg.setAttributes(&I, Idx);
8518
8519 // If we have an explicit sret argument that is an Instruction, (i.e., it
8520 // might point to function-local memory), we can't meaningfully tail-call.
8521 if (Arg.IsSRet && isa<Instruction>(I.getArgOperand(Idx)))
8522 isTailCall = false;
8523
8524 Args.push_back(Arg);
8525 }
8526
8527 SDValue ConvControlToken;
8528 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8529 auto *Token = Bundle->Inputs[0].get();
8530 ConvControlToken = getValue(Token);
8531 }
8532
8533 TargetLowering::CallLoweringInfo CLI(DAG);
8534 CLI.setDebugLoc(getCurSDLoc())
8535 .setChain(getRoot())
8536 .setCallee(CallingConv::AMDGPU_Gfx_WholeWave, I.getType(),
8537 getValue(I.getArgOperand(0)), std::move(Args))
8538 .setTailCall(isTailCall && canTailCall(I))
8539 .setIsPreallocated(
8540 I.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
8541 .setConvergent(I.isConvergent())
8542 .setConvergenceControlToken(ConvControlToken);
8543 CLI.CB = &I;
8544
8545 std::pair<SDValue, SDValue> Result =
8546 lowerInvokable(CLI, /*EHPadBB=*/nullptr);
8547
8548 if (Result.first.getNode())
8549 setValue(&I, Result.first);
8550 return;
8551 }
8552 case Intrinsic::ptrmask: {
8553 SDValue Ptr = getValue(I.getOperand(0));
8554 SDValue Mask = getValue(I.getOperand(1));
8555
8556 // On arm64_32, pointers are 32 bits when stored in memory, but
8557 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
8558 // match the index type, but the pointer is 64 bits, so the mask must be
8559 // zero-extended up to 64 bits to match the pointer.
8560 EVT PtrVT =
8561 TLI.getValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8562 EVT MemVT =
8563 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8564 assert(PtrVT == Ptr.getValueType());
8565 if (Mask.getValueType().getFixedSizeInBits() < MemVT.getFixedSizeInBits()) {
8566 // For AMDGPU buffer descriptors the mask is 48 bits, but the pointer is
8567 // 128-bit, so we have to pad the mask with ones for unused bits.
8568 auto HighOnes = DAG.getNode(
8569 ISD::SHL, sdl, PtrVT, DAG.getAllOnesConstant(sdl, PtrVT),
8570 DAG.getShiftAmountConstant(Mask.getValueType().getFixedSizeInBits(),
8571 PtrVT, sdl));
8572 Mask = DAG.getNode(ISD::OR, sdl, PtrVT,
8573 DAG.getZExtOrTrunc(Mask, sdl, PtrVT), HighOnes);
8574 } else if (Mask.getValueType() != PtrVT)
8575 Mask = DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);
8576
8577 assert(Mask.getValueType() == PtrVT);
8578 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
8579 return;
8580 }
8581 case Intrinsic::threadlocal_address: {
8582 setValue(&I, getValue(I.getOperand(0)));
8583 return;
8584 }
8585 case Intrinsic::get_active_lane_mask: {
8586 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8587 SDValue Index = getValue(I.getOperand(0));
8588 SDValue TripCount = getValue(I.getOperand(1));
8589 EVT ElementVT = Index.getValueType();
8590
8591 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
8592 setValue(&I, DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, sdl, CCVT, Index,
8593 TripCount));
8594 return;
8595 }
8596
8597 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
8598 CCVT.getVectorElementCount());
8599
8600 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
8601 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
8602 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
8603 SDValue VectorInduction = DAG.getNode(
8604 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
8605 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
8606 VectorTripCount, ISD::CondCode::SETULT);
8607 setValue(&I, SetCC);
8608 return;
8609 }
8610 case Intrinsic::mask_beforefirst: {
8611 SDValue Op = getValue(I.getOperand(0));
8612 setValue(&I,
8613 DAG.getNode(ISD::MASK_BEFOREFIRST, sdl, Op.getValueType(), Op));
8614 return;
8615 }
8616 case Intrinsic::experimental_get_vector_length: {
8617 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8618 "Expected positive VF");
8619 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
8620 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
8621
8622 SDValue Count = getValue(I.getOperand(0));
8623 EVT CountVT = Count.getValueType();
8624
8625 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8626 visitTargetIntrinsic(I, Intrinsic);
8627 return;
8628 }
8629
8630 // Expand to a umin between the trip count and the maximum elements the type
8631 // can hold.
8632 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8633
8634 // Extend the trip count to at least the result VT.
8635 if (CountVT.bitsLT(VT)) {
8636 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
8637 CountVT = VT;
8638 }
8639
8640 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
8642
8643 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
8644 // Clip to the result type if needed.
8645 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
8646
8647 setValue(&I, Trunc);
8648 return;
8649 }
8650 case Intrinsic::vector_partial_reduce_add: {
8651 SDValue Acc = getValue(I.getOperand(0));
8652 SDValue Input = getValue(I.getOperand(1));
8653 setValue(&I,
8654 DAG.getNode(ISD::PARTIAL_REDUCE_UMLA, sdl, Acc.getValueType(), Acc,
8655 Input, DAG.getConstant(1, sdl, Input.getValueType())));
8656 return;
8657 }
8658 case Intrinsic::vector_partial_reduce_fadd: {
8659 SDNodeFlags Flags;
8660 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
8661 Flags.copyFMF(*FPOp);
8662 SDValue Acc = getValue(I.getOperand(0));
8663 SDValue Input = getValue(I.getOperand(1));
8664 setValue(&I, DAG.getNode(ISD::PARTIAL_REDUCE_FMLA, sdl, Acc.getValueType(),
8665 Acc, Input,
8666 DAG.getConstantFP(1.0, sdl, Input.getValueType()),
8667 Flags));
8668 return;
8669 }
8670 case Intrinsic::experimental_cttz_elts: {
8671 SDValue Op = getValue(I.getOperand(0));
8672 EVT OpVT = Op.getValueType();
8673 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
8674 bool ZeroIsPoison =
8675 !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
8676 if (OpVT.getVectorElementType() != MVT::i1) {
8677 // Compare the input vector elements to zero & use to count trailing
8678 // zeros.
8679 SDValue AllZero = DAG.getConstant(0, sdl, OpVT);
8680 EVT I1OpVT = OpVT.changeVectorElementType(*DAG.getContext(), MVT::i1);
8681 Op = DAG.getSetCC(sdl, I1OpVT, Op, AllZero, ISD::SETNE);
8682 }
8683 setValue(&I, DAG.getNode(ZeroIsPoison ? ISD::CTTZ_ELTS_ZERO_POISON
8685 sdl, RetTy, Op));
8686 return;
8687 }
8688 case Intrinsic::vector_insert: {
8689 SDValue Vec = getValue(I.getOperand(0));
8690 SDValue SubVec = getValue(I.getOperand(1));
8691 SDValue Index = getValue(I.getOperand(2));
8692
8693 // The intrinsic's index type is i64, but the SDNode requires an index type
8694 // suitable for the target. Convert the index as required.
8695 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8696 if (Index.getValueType() != VectorIdxTy)
8697 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8698
8699 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8700 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
8701 Index));
8702 return;
8703 }
8704 case Intrinsic::vector_extract: {
8705 SDValue Vec = getValue(I.getOperand(0));
8706 SDValue Index = getValue(I.getOperand(1));
8707 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8708
8709 // The intrinsic's index type is i64, but the SDNode requires an index type
8710 // suitable for the target. Convert the index as required.
8711 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8712 if (Index.getValueType() != VectorIdxTy)
8713 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8714
8715 setValue(&I,
8716 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
8717 return;
8718 }
8719 case Intrinsic::experimental_vector_match: {
8720 SDValue Op1 = getValue(I.getOperand(0));
8721 SDValue Op2 = getValue(I.getOperand(1));
8722 SDValue Mask = getValue(I.getOperand(2));
8723 EVT ResVT = Mask.getValueType();
8724 setValue(&I, DAG.getNode(ISD::VECTOR_MATCH, sdl, ResVT, Op1, Op2, Mask));
8725 return;
8726 }
8727 case Intrinsic::vector_reverse:
8728 visitVectorReverse(I);
8729 return;
8730 case Intrinsic::vector_splice_left:
8731 case Intrinsic::vector_splice_right:
8732 visitVectorSplice(I);
8733 return;
8734 case Intrinsic::callbr_landingpad:
8735 visitCallBrLandingPad(I);
8736 return;
8737 case Intrinsic::vector_interleave2:
8738 visitVectorInterleave(I, 2);
8739 return;
8740 case Intrinsic::vector_interleave3:
8741 visitVectorInterleave(I, 3);
8742 return;
8743 case Intrinsic::vector_interleave4:
8744 visitVectorInterleave(I, 4);
8745 return;
8746 case Intrinsic::vector_interleave5:
8747 visitVectorInterleave(I, 5);
8748 return;
8749 case Intrinsic::vector_interleave6:
8750 visitVectorInterleave(I, 6);
8751 return;
8752 case Intrinsic::vector_interleave7:
8753 visitVectorInterleave(I, 7);
8754 return;
8755 case Intrinsic::vector_interleave8:
8756 visitVectorInterleave(I, 8);
8757 return;
8758 case Intrinsic::vector_deinterleave2:
8759 visitVectorDeinterleave(I, 2);
8760 return;
8761 case Intrinsic::vector_deinterleave3:
8762 visitVectorDeinterleave(I, 3);
8763 return;
8764 case Intrinsic::vector_deinterleave4:
8765 visitVectorDeinterleave(I, 4);
8766 return;
8767 case Intrinsic::vector_deinterleave5:
8768 visitVectorDeinterleave(I, 5);
8769 return;
8770 case Intrinsic::vector_deinterleave6:
8771 visitVectorDeinterleave(I, 6);
8772 return;
8773 case Intrinsic::vector_deinterleave7:
8774 visitVectorDeinterleave(I, 7);
8775 return;
8776 case Intrinsic::vector_deinterleave8:
8777 visitVectorDeinterleave(I, 8);
8778 return;
8779 case Intrinsic::vector_repeat: {
8780 SDValue Vec = getValue(I.getOperand(0));
8781 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8782 setValue(&I, DAG.getNode(ISD::VECTOR_REPEAT, sdl, ResultVT, Vec));
8783 return;
8784 }
8785 case Intrinsic::experimental_vector_compress:
8786 setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,
8787 getValue(I.getArgOperand(0)).getValueType(),
8788 getValue(I.getArgOperand(0)),
8789 getValue(I.getArgOperand(1)),
8790 getValue(I.getArgOperand(2)), Flags));
8791 return;
8792 case Intrinsic::vector_shuffle:
8793 setValue(&I, DAG.getNode(ISD::VECTOR_SHUFFLE_VAR, sdl,
8794 TLI.getValueType(DAG.getDataLayout(), I.getType()),
8795 getValue(I.getArgOperand(0)),
8796 getValue(I.getArgOperand(1)), Flags));
8797 return;
8798 case Intrinsic::experimental_convergence_anchor:
8799 case Intrinsic::experimental_convergence_entry:
8800 case Intrinsic::experimental_convergence_loop:
8801 visitConvergenceControl(I, Intrinsic);
8802 return;
8803 case Intrinsic::experimental_vector_histogram_add: {
8804 visitVectorHistogram(I, Intrinsic);
8805 return;
8806 }
8807 case Intrinsic::experimental_vector_extract_last_active: {
8808 visitVectorExtractLastActive(I, Intrinsic);
8809 return;
8810 }
8811 case Intrinsic::loop_dependence_war_mask:
8812 setValue(&I,
8814 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),
8815 getValue(I.getOperand(1)), getValue(I.getOperand(2)),
8816 DAG.getConstant(0, sdl, MVT::i64)));
8817 return;
8818 case Intrinsic::loop_dependence_raw_mask:
8819 setValue(&I,
8821 EVT::getEVT(I.getType()), getValue(I.getOperand(0)),
8822 getValue(I.getOperand(1)), getValue(I.getOperand(2)),
8823 DAG.getConstant(0, sdl, MVT::i64)));
8824 return;
8825 case Intrinsic::masked_udiv:
8826 setValue(&I,
8827 DAG.getNode(ISD::MASKED_UDIV, sdl, EVT::getEVT(I.getType()),
8828 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8829 getValue(I.getOperand(2))));
8830 return;
8831 case Intrinsic::masked_sdiv:
8832 setValue(&I,
8833 DAG.getNode(ISD::MASKED_SDIV, sdl, EVT::getEVT(I.getType()),
8834 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8835 getValue(I.getOperand(2))));
8836 return;
8837 case Intrinsic::masked_urem:
8838 setValue(&I,
8839 DAG.getNode(ISD::MASKED_UREM, sdl, EVT::getEVT(I.getType()),
8840 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8841 getValue(I.getOperand(2))));
8842 return;
8843 case Intrinsic::masked_srem:
8844 setValue(&I,
8845 DAG.getNode(ISD::MASKED_SREM, sdl, EVT::getEVT(I.getType()),
8846 getValue(I.getOperand(0)), getValue(I.getOperand(1)),
8847 getValue(I.getOperand(2))));
8848 return;
8849 }
8850}
8851
8852void SelectionDAGBuilder::pushFPOpOutChain(SDValue Result,
8854 assert(Result.getNode()->getNumValues() == 2);
8855 SDValue OutChain = Result.getValue(1);
8856 assert(OutChain.getValueType() == MVT::Other);
8857
8858 // Instead of updating the root immediately, push the produced chain to the
8859 // appropriate list, deferring the update until the root is requested. In this
8860 // case, the nodes from the lists are chained using TokenFactor, indicating
8861 // that the operations are independent.
8862 //
8863 // In particular, the root is updated before any call that might access the
8864 // floating-point environment, except for constrained intrinsics.
8865 switch (EB) {
8868 PendingConstrainedFP.push_back(OutChain);
8869 break;
8871 PendingConstrainedFPStrict.push_back(OutChain);
8872 break;
8873 }
8874}
8875
8876void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8877 const ConstrainedFPIntrinsic &FPI) {
8878 SDLoc sdl = getCurSDLoc();
8879
8880 // We do not need to serialize constrained FP intrinsics against
8881 // each other or against (nonvolatile) loads, so they can be
8882 // chained like loads.
8884 SDValue Chain = getFPOperationRoot(EB);
8886 Opers.push_back(Chain);
8887 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8888 Opers.push_back(getValue(FPI.getArgOperand(I)));
8889
8890 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8891 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
8892 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
8893
8894 SDNodeFlags Flags;
8896 Flags.setNoFPExcept(true);
8897
8898 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
8899 Flags.copyFMF(*FPOp);
8900
8901 unsigned Opcode;
8902 switch (FPI.getIntrinsicID()) {
8903 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8904#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8905 case Intrinsic::INTRINSIC: \
8906 Opcode = ISD::STRICT_##DAGN; \
8907 break;
8908#include "llvm/IR/ConstrainedOps.def"
8909 case Intrinsic::experimental_constrained_fmuladd: {
8910 Opcode = ISD::STRICT_FMA;
8911 // Break fmuladd into fmul and fadd.
8912 if (!TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
8913 Opers.pop_back();
8914 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8915 pushFPOpOutChain(Mul, EB);
8916 Opcode = ISD::STRICT_FADD;
8917 Opers.clear();
8918 Opers.push_back(Mul.getValue(1));
8919 Opers.push_back(Mul.getValue(0));
8920 Opers.push_back(getValue(FPI.getArgOperand(2)));
8921 }
8922 break;
8923 }
8924 }
8925
8926 // A few strict DAG nodes carry additional operands that are not
8927 // set up by the default code above.
8928 switch (Opcode) {
8929 default: break;
8931 Opers.push_back(
8932 DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
8933 break;
8934 case ISD::STRICT_FSETCC:
8935 case ISD::STRICT_FSETCCS: {
8936 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8937 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8938 if (DAG.isKnownNeverNaN(Opers[1]) && DAG.isKnownNeverNaN(Opers[2]))
8939 Condition = getFCmpCodeWithoutNaN(Condition);
8940 Opers.push_back(DAG.getCondCode(Condition));
8941 break;
8942 }
8943 }
8944
8945 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8946 pushFPOpOutChain(Result, EB);
8947
8948 SDValue FPResult = Result.getValue(0);
8949 setValue(&FPI, FPResult);
8950}
8951
8952static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8953 std::optional<unsigned> ResOPC;
8954 switch (VPIntrin.getIntrinsicID()) {
8955 case Intrinsic::vp_cttz_elts: {
8956 bool IsZeroPoison = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8957 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_POISON : ISD::VP_CTTZ_ELTS;
8958 break;
8959 }
8960#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8961 case Intrinsic::VPID: \
8962 ResOPC = ISD::VPSD; \
8963 break;
8964#include "llvm/IR/VPIntrinsics.def"
8965 }
8966
8967 if (!ResOPC)
8969 "Inconsistency: no SDNode available for this VPIntrinsic!");
8970
8971 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8972 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8973 if (VPIntrin.getFastMathFlags().allowReassoc())
8974 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8975 : ISD::VP_REDUCE_FMUL;
8976 }
8977
8978 return *ResOPC;
8979}
8980
8981void SelectionDAGBuilder::visitVPLoad(
8982 const VPIntrinsic &VPIntrin, EVT VT,
8983 const SmallVectorImpl<SDValue> &OpValues) {
8984 SDLoc DL = getCurSDLoc();
8985 Value *PtrOperand = VPIntrin.getArgOperand(0);
8986 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8987 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8988 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8989 SDValue LD;
8990 // Do not serialize variable-length loads of constant memory with
8991 // anything.
8992 if (!Alignment)
8993 Alignment = DAG.getEVTAlign(VT);
8994 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8995 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
8996 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8997 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8998 MachineMemOperand::Flags MMOFlags =
9000 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9001 MachinePointerInfo(PtrOperand), MMOFlags,
9003 MMOMetadata(AAInfo, Ranges));
9004 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9005 MMO, false /*IsExpanding */);
9006 if (AddToChain)
9007 PendingLoads.push_back(LD.getValue(1));
9008 setValue(&VPIntrin, LD);
9009}
9010
9011void SelectionDAGBuilder::visitVPLoadFF(
9012 const VPIntrinsic &VPIntrin, EVT VT, EVT EVLVT,
9013 const SmallVectorImpl<SDValue> &OpValues) {
9014 assert(OpValues.size() == 3 && "Unexpected number of operands");
9015 SDLoc DL = getCurSDLoc();
9016 Value *PtrOperand = VPIntrin.getArgOperand(0);
9017 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9018 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9019 const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
9020 SDValue LD;
9021 // Do not serialize variable-length loads of constant memory with
9022 // anything.
9023 if (!Alignment)
9024 Alignment = DAG.getEVTAlign(VT);
9025 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
9026 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
9027 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9028 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9029 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
9031 MMOMetadata(AAInfo, Ranges));
9032 LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9033 MMO);
9034 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));
9035 if (AddToChain)
9036 PendingLoads.push_back(LD.getValue(2));
9037 setValue(&VPIntrin, DAG.getMergeValues({LD.getValue(0), Trunc}, DL));
9038}
9039
9040void SelectionDAGBuilder::visitVPGather(
9041 const VPIntrinsic &VPIntrin, EVT VT,
9042 const SmallVectorImpl<SDValue> &OpValues) {
9043 SDLoc DL = getCurSDLoc();
9044 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9045 Value *PtrOperand = VPIntrin.getArgOperand(0);
9046 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9047 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9048 const MDNode *Ranges = getRangeMetadata(VPIntrin);
9049 SDValue LD;
9050 if (!Alignment)
9051 Alignment = DAG.getEVTAlign(VT.getScalarType());
9052 unsigned AS =
9053 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9054 MachineMemOperand::Flags MMOFlags =
9056 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9057 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9058 *Alignment, MMOMetadata(AAInfo, Ranges));
9059 SDValue Base, Index, Scale;
9060 bool UniformBase =
9061 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
9062 VT.getScalarStoreSize());
9063 if (!UniformBase) {
9064 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
9065 Index = getValue(PtrOperand);
9066 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
9067 }
9068 EVT IdxVT = Index.getValueType();
9069 EVT EltTy = IdxVT.getVectorElementType();
9070 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
9071 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
9072 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
9073 }
9074 LD = DAG.getGatherVP(
9075 DAG.getVTList(VT, MVT::Other), VT, DL,
9076 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
9078 PendingLoads.push_back(LD.getValue(1));
9079 setValue(&VPIntrin, LD);
9080}
9081
9082void SelectionDAGBuilder::visitVPStore(
9083 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9084 SDLoc DL = getCurSDLoc();
9085 Value *PtrOperand = VPIntrin.getArgOperand(1);
9086 EVT VT = OpValues[0].getValueType();
9087 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9088 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9089 SDValue ST;
9090 if (!Alignment)
9091 Alignment = DAG.getEVTAlign(VT);
9092 SDValue Ptr = OpValues[1];
9093 SDValue Offset = DAG.getPOISON(Ptr.getValueType());
9094 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9095 MachineMemOperand::Flags MMOFlags =
9097 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9098 MachinePointerInfo(PtrOperand), MMOFlags,
9099 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
9100 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
9101 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
9102 /* IsTruncating */ false, /*IsCompressing*/ false);
9103 DAG.setRoot(ST);
9104 setValue(&VPIntrin, ST);
9105}
9106
9107void SelectionDAGBuilder::visitVPScatter(
9108 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9109 SDLoc DL = getCurSDLoc();
9110 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9111 Value *PtrOperand = VPIntrin.getArgOperand(1);
9112 EVT VT = OpValues[0].getValueType();
9113 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9114 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9115 SDValue ST;
9116 if (!Alignment)
9117 Alignment = DAG.getEVTAlign(VT.getScalarType());
9118 unsigned AS =
9119 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
9120 MachineMemOperand::Flags MMOFlags =
9122 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9123 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9124 *Alignment, AAInfo);
9125 SDValue Base, Index, Scale;
9126 bool UniformBase =
9127 getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
9128 VT.getScalarStoreSize());
9129 if (!UniformBase) {
9130 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
9131 Index = getValue(PtrOperand);
9132 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
9133 }
9134 EVT IdxVT = Index.getValueType();
9135 EVT EltTy = IdxVT.getVectorElementType();
9136 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
9137 EVT NewIdxVT = IdxVT.changeVectorElementType(*DAG.getContext(), EltTy);
9138 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
9139 }
9140 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
9141 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
9142 OpValues[2], OpValues[3]},
9143 MMO, ISD::SIGNED_SCALED);
9144 DAG.setRoot(ST);
9145 setValue(&VPIntrin, ST);
9146}
9147
9148void SelectionDAGBuilder::visitVPStridedLoad(
9149 const VPIntrinsic &VPIntrin, EVT VT,
9150 const SmallVectorImpl<SDValue> &OpValues) {
9151 SDLoc DL = getCurSDLoc();
9152 Value *PtrOperand = VPIntrin.getArgOperand(0);
9153 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9154 if (!Alignment)
9155 Alignment = DAG.getEVTAlign(VT.getScalarType());
9156 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9157 const MDNode *Ranges = getRangeMetadata(VPIntrin);
9158 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
9159 bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
9160 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
9161 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9162 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9163 MachineMemOperand::Flags MMOFlags =
9165 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9166 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9167 *Alignment, MMOMetadata(AAInfo, Ranges));
9168
9169 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
9170 OpValues[2], OpValues[3], MMO,
9171 false /*IsExpanding*/);
9172
9173 if (AddToChain)
9174 PendingLoads.push_back(LD.getValue(1));
9175 setValue(&VPIntrin, LD);
9176}
9177
9178void SelectionDAGBuilder::visitVPStridedStore(
9179 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
9180 SDLoc DL = getCurSDLoc();
9181 Value *PtrOperand = VPIntrin.getArgOperand(1);
9182 EVT VT = OpValues[0].getValueType();
9183 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
9184 if (!Alignment)
9185 Alignment = DAG.getEVTAlign(VT.getScalarType());
9186 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
9187 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
9188 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9189 MachineMemOperand::Flags MMOFlags =
9191 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
9192 MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
9193 *Alignment, AAInfo);
9194
9195 SDValue ST = DAG.getStridedStoreVP(
9196 getMemoryRoot(), DL, OpValues[0], OpValues[1],
9197 DAG.getPOISON(OpValues[1].getValueType()), OpValues[2], OpValues[3],
9198 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
9199 /*IsCompressing*/ false);
9200
9201 DAG.setRoot(ST);
9202 setValue(&VPIntrin, ST);
9203}
9204
9205void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
9206 const VPIntrinsic &VPIntrin) {
9207 SDLoc DL = getCurSDLoc();
9208 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
9209
9210 auto IID = VPIntrin.getIntrinsicID();
9211
9212 SmallVector<EVT, 4> ValueVTs;
9213 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9214 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
9215 SDVTList VTs = DAG.getVTList(ValueVTs);
9216
9217 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
9218
9219 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
9220 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
9221 "Unexpected target EVL type");
9222
9223 // Request operands.
9224 SmallVector<SDValue, 7> OpValues;
9225 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
9226 auto Op = getValue(VPIntrin.getArgOperand(I));
9227 if (I == EVLParamPos)
9228 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
9229 OpValues.push_back(Op);
9230 }
9231
9232 switch (Opcode) {
9233 default: {
9234 SDNodeFlags SDFlags;
9235 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
9236 SDFlags.copyFMF(*FPMO);
9237 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
9238 setValue(&VPIntrin, Result);
9239 break;
9240 }
9241 case ISD::VP_LOAD:
9242 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
9243 break;
9244 case ISD::VP_LOAD_FF:
9245 visitVPLoadFF(VPIntrin, ValueVTs[0], ValueVTs[1], OpValues);
9246 break;
9247 case ISD::VP_GATHER:
9248 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
9249 break;
9250 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9251 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
9252 break;
9253 case ISD::VP_STORE:
9254 visitVPStore(VPIntrin, OpValues);
9255 break;
9256 case ISD::VP_SCATTER:
9257 visitVPScatter(VPIntrin, OpValues);
9258 break;
9259 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9260 visitVPStridedStore(VPIntrin, OpValues);
9261 break;
9262 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
9263 case ISD::VP_CTTZ_ELTS: {
9264 SDValue Result =
9265 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
9266 setValue(&VPIntrin, Result);
9267 break;
9268 }
9269 }
9270}
9271
9273 const BasicBlock *EHPadBB,
9274 MCSymbol *&BeginLabel) {
9275 MachineFunction &MF = DAG.getMachineFunction();
9276
9277 // Skip emitting EH_LABEL on targets whose exception tables don't reference
9278 // them (32-bit x86 SEH, Wasm).
9280 FuncInfo.ExceptionModel)) {
9281 BeginLabel = nullptr;
9282 return Chain;
9283 }
9284
9285 // Insert a label before the invoke call to mark the try range. This can be
9286 // used to detect deletion of the invoke via the MachineModuleInfo.
9287 BeginLabel = MF.getContext().createTempSymbol();
9288
9289 // For SjLj, keep track of which landing pads go with which invokes
9290 // so as to maintain the ordering of pads in the LSDA.
9291 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
9292 if (CallSiteIndex) {
9293 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
9294 LPadToCallSiteMap[FuncInfo.getMBB(EHPadBB)].push_back(CallSiteIndex);
9295
9296 // Now that the call site is handled, stop tracking it.
9297 FuncInfo.setCurrentCallSite(0);
9298 }
9299
9300 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
9301}
9302
9303SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
9304 const BasicBlock *EHPadBB,
9305 MCSymbol *BeginLabel) {
9306 // No labels were emitted.
9307 if (!BeginLabel)
9308 return Chain;
9309
9311
9312 // Insert a label at the end of the invoke call to mark the try range. This
9313 // can be used to detect deletion of the invoke via the MachineModuleInfo.
9314 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
9315 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
9316
9317 // Inform MachineModuleInfo of range.
9319 // There is a platform (e.g. wasm) that uses funclet style IR but does not
9320 // actually use outlined funclets and their LSDA info style.
9321 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
9322 assert(II && "II should've been set");
9323 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
9324 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
9325 } else if (!isScopedEHPersonality(Pers)) {
9326 assert(EHPadBB);
9327 MF.addInvoke(FuncInfo.getMBB(EHPadBB), BeginLabel, EndLabel);
9328 }
9329
9330 return Chain;
9331}
9332
9333std::pair<SDValue, SDValue>
9335 const BasicBlock *EHPadBB) {
9336 MCSymbol *BeginLabel = nullptr;
9337
9338 if (EHPadBB) {
9339 // Both PendingLoads and PendingExports must be flushed here;
9340 // this call might not return.
9341 (void)getRoot();
9342 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
9343 CLI.setChain(getRoot());
9344 }
9345
9346 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9347 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
9348
9349 assert((CLI.IsTailCall || Result.second.getNode()) &&
9350 "Non-null chain expected with non-tail call!");
9351 assert((Result.second.getNode() || !Result.first.getNode()) &&
9352 "Null value expected with tail call!");
9353
9354 if (!Result.second.getNode()) {
9355 // As a special case, a null chain means that a tail call has been emitted
9356 // and the DAG root is already updated.
9357 HasTailCall = true;
9358
9359 // Since there's no actual continuation from this block, nothing can be
9360 // relying on us setting vregs for them.
9361 PendingExports.clear();
9362 } else {
9363 DAG.setRoot(Result.second);
9364 }
9365
9366 if (EHPadBB) {
9367 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
9368 BeginLabel));
9369 Result.second = getRoot();
9370 }
9371
9372 return Result;
9373}
9374
9376 bool isMustTailCall = CB.isMustTailCall();
9377
9378 // Avoid emitting tail calls in functions with the disable-tail-calls
9379 // attribute.
9380 const Function *Caller = CB.getParent()->getParent();
9381 if (!isMustTailCall &&
9382 Caller->getFnAttribute("disable-tail-calls").getValueAsBool())
9383 return false;
9384
9385 // We can't tail call inside a function with a swifterror argument. Lowering
9386 // does not support this yet. It would have to move into the swifterror
9387 // register before the call.
9388 if (DAG.hasSwiftErrorArg())
9389 return false;
9390
9391 // Check if target-independent constraints permit a tail call here.
9392 // Target-dependent constraints are checked within TLI->LowerCallTo.
9393 return isInTailCallPosition(CB, DAG.getTarget());
9394}
9395
9397 bool isTailCall, bool isMustTailCall,
9398 const BasicBlock *EHPadBB,
9399 const TargetLowering::PtrAuthInfo *PAI) {
9400 auto &DL = DAG.getDataLayout();
9401 FunctionType *FTy = CB.getFunctionType();
9402 Type *RetTy = CB.getType();
9403
9405 Args.reserve(CB.arg_size());
9406
9407 const Value *SwiftErrorVal = nullptr;
9408 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9409
9410 if (isTailCall)
9411 isTailCall = canTailCall(CB);
9412
9413 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
9414 const Value *V = *I;
9415
9416 // Skip empty types
9417 if (V->getType()->isEmptyTy())
9418 continue;
9419
9420 SDValue ArgNode = getValue(V);
9421 TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
9422 Entry.setAttributes(&CB, I - CB.arg_begin());
9423
9424 // Use swifterror virtual register as input to the call.
9425 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
9426 SwiftErrorVal = V;
9427 // We find the virtual register for the actual swifterror argument.
9428 // Instead of using the Value, we use the virtual register instead.
9429 Entry.Node =
9430 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
9431 EVT(TLI.getPointerTy(DL)));
9432 }
9433
9434 Args.push_back(Entry);
9435
9436 // If we have an explicit sret argument that is an Instruction, (i.e., it
9437 // might point to function-local memory), we can't meaningfully tail-call.
9438 if (Entry.IsSRet && isa<Instruction>(V))
9439 isTailCall = false;
9440 }
9441
9442 // If call site has a cfguardtarget operand bundle, create and add an
9443 // additional ArgListEntry.
9444 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
9445 Value *V = Bundle->Inputs[0];
9447 Entry.IsCFGuardTarget = true;
9448 Args.push_back(Entry);
9449 }
9450
9451 // Disable tail calls if there is an swifterror argument. Targets have not
9452 // been updated to support tail calls.
9453 if (TLI.supportSwiftError() && SwiftErrorVal)
9454 isTailCall = false;
9455
9456 ConstantInt *CFIType = nullptr;
9457 if (CB.isIndirectCall()) {
9458 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
9459 if (!TLI.supportKCFIBundles())
9461 "Target doesn't support calls with kcfi operand bundles.");
9462 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
9463 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
9464 }
9465 }
9466
9467 SDValue ConvControlToken;
9468 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
9469 auto *Token = Bundle->Inputs[0].get();
9470 ConvControlToken = getValue(Token);
9471 }
9472
9473 GlobalValue *DeactivationSymbol = nullptr;
9475 DeactivationSymbol = cast<GlobalValue>(Bundle->Inputs[0].get());
9476 }
9477
9480 .setChain(getRoot())
9481 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
9482 .setTailCall(isTailCall)
9486 .setCFIType(CFIType)
9487 .setConvergenceControlToken(ConvControlToken)
9488 .setDeactivationSymbol(DeactivationSymbol);
9489
9490 // Set the pointer authentication info if we have it.
9491 if (PAI) {
9492 if (!TLI.supportPtrAuthBundles())
9494 "This target doesn't support calls with ptrauth operand bundles.");
9495 CLI.setPtrAuth(*PAI);
9496 }
9497
9498 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
9499
9500 if (Result.first.getNode()) {
9501 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
9502 Result.first = lowerNoFPClassToAssertNoFPClass(DAG, CB, Result.first);
9503 setValue(&CB, Result.first);
9504 }
9505
9506 // The last element of CLI.InVals has the SDValue for swifterror return.
9507 // Here we copy it to a virtual register and update SwiftErrorMap for
9508 // book-keeping.
9509 if (SwiftErrorVal && TLI.supportSwiftError()) {
9510 // Get the last element of InVals.
9511 SDValue Src = CLI.InVals.back();
9512 Register VReg =
9513 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
9514 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
9515 DAG.setRoot(CopyNode);
9516 }
9517}
9518
9519static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
9520 SelectionDAGBuilder &Builder) {
9521 // Check to see if this load can be trivially constant folded, e.g. if the
9522 // input is from a string literal.
9523 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
9524 // Cast pointer to the type we really want to load.
9525 Type *LoadTy =
9526 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
9527 if (LoadVT.isVector())
9528 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
9529 if (const Constant *LoadCst =
9530 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
9531 LoadTy, Builder.DAG.getDataLayout()))
9532 return Builder.getValue(LoadCst);
9533 }
9534
9535 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9536 // still constant memory, the input chain can be the entry node.
9537 SDValue Root;
9538 bool ConstantMemory = false;
9539
9540 // Do not serialize (non-volatile) loads of constant memory with anything.
9541 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) {
9542 Root = Builder.DAG.getEntryNode();
9543 ConstantMemory = true;
9544 } else {
9545 // Do not serialize non-volatile loads against each other.
9546 Root = Builder.DAG.getRoot();
9547 }
9548
9549 SDValue Ptr = Builder.getValue(PtrVal);
9550 SDValue LoadVal =
9551 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
9552 MachinePointerInfo(PtrVal), Align(1));
9553
9554 if (!ConstantMemory)
9555 Builder.PendingLoads.push_back(LoadVal.getValue(1));
9556 return LoadVal;
9557}
9558
9559/// Record the value for an instruction that produces an integer result,
9560/// converting the type where necessary.
9561void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9562 SDValue Value,
9563 bool IsSigned) {
9564 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
9565 I.getType(), true);
9566 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
9567 setValue(&I, Value);
9568}
9569
9570/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9571/// true and lower it. Otherwise return false, and it will be lowered like a
9572/// normal call.
9573/// The caller already checked that \p I calls the appropriate LibFunc with a
9574/// correct prototype.
9575bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9576 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
9577 const Value *Size = I.getArgOperand(2);
9578 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
9579 if (CSize && CSize->getZExtValue() == 0) {
9580 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
9581 I.getType(), true);
9582 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
9583 return true;
9584 }
9585
9586 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9587 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9588 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
9589 getValue(Size), &I);
9590 if (Res.first.getNode()) {
9591 processIntegerCallValue(I, Res.first, true);
9592 PendingLoads.push_back(Res.second);
9593 return true;
9594 }
9595
9596 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9597 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9598 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
9599 return false;
9600
9601 // If the target has a fast compare for the given size, it will return a
9602 // preferred load type for that size. Require that the load VT is legal and
9603 // that the target supports unaligned loads of that type. Otherwise, return
9604 // INVALID.
9605 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9606 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9607 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9608 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9609 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9610 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9611 // TODO: Check alignment of src and dest ptrs.
9612 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9613 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9614 if (!TLI.isTypeLegal(LVT) ||
9615 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
9618 }
9619
9620 return LVT;
9621 };
9622
9623 // This turns into unaligned loads. We only do this if the target natively
9624 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9625 // we'll only produce a small number of byte loads.
9626 MVT LoadVT;
9627 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9628 switch (NumBitsToCompare) {
9629 default:
9630 return false;
9631 case 16:
9632 LoadVT = MVT::i16;
9633 break;
9634 case 32:
9635 LoadVT = MVT::i32;
9636 break;
9637 case 64:
9638 case 128:
9639 case 256:
9640 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9641 break;
9642 }
9643
9644 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9645 return false;
9646
9647 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
9648 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
9649
9650 // Bitcast to a wide integer type if the loads are vectors.
9651 if (LoadVT.isVector()) {
9652 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
9653 LoadL = DAG.getBitcast(CmpVT, LoadL);
9654 LoadR = DAG.getBitcast(CmpVT, LoadR);
9655 }
9656
9657 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
9658 processIntegerCallValue(I, Cmp, false);
9659 return true;
9660}
9661
9662/// See if we can lower a memchr call into an optimized form. If so, return
9663/// true and lower it. Otherwise return false, and it will be lowered like a
9664/// normal call.
9665/// The caller already checked that \p I calls the appropriate LibFunc with a
9666/// correct prototype.
9667bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9668 const Value *Src = I.getArgOperand(0);
9669 const Value *Char = I.getArgOperand(1);
9670 const Value *Length = I.getArgOperand(2);
9671
9672 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9673 std::pair<SDValue, SDValue> Res =
9674 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
9675 getValue(Src), getValue(Char), getValue(Length),
9676 MachinePointerInfo(Src));
9677 if (Res.first.getNode()) {
9678 setValue(&I, Res.first);
9679 PendingLoads.push_back(Res.second);
9680 return true;
9681 }
9682
9683 return false;
9684}
9685
9686/// See if we can lower a memccpy call into an optimized form. If so, return
9687/// true and lower it, otherwise return false and it will be lowered like a
9688/// normal call.
9689/// The caller already checked that \p I calls the appropriate LibFunc with a
9690/// correct prototype.
9691bool SelectionDAGBuilder::visitMemCCpyCall(const CallInst &I) {
9692 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9693 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemccpy(
9694 DAG, getCurSDLoc(), DAG.getRoot(), getValue(I.getArgOperand(0)),
9695 getValue(I.getArgOperand(1)), getValue(I.getArgOperand(2)),
9696 getValue(I.getArgOperand(3)), &I);
9697
9698 if (Res.first) {
9699 processIntegerCallValue(I, Res.first, true);
9700 PendingLoads.push_back(Res.second);
9701 return true;
9702 }
9703 return false;
9704}
9705
9706/// See if we can lower a mempcpy call into an optimized form. If so, return
9707/// true and lower it. Otherwise return false, and it will be lowered like a
9708/// normal call.
9709/// The caller already checked that \p I calls the appropriate LibFunc with a
9710/// correct prototype.
9711bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9712 SDValue Dst = getValue(I.getArgOperand(0));
9713 SDValue Src = getValue(I.getArgOperand(1));
9714 SDValue Size = getValue(I.getArgOperand(2));
9715
9716 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
9717 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
9718
9719 SDLoc sdl = getCurSDLoc();
9720
9721 // In the mempcpy context we need to pass in a false value for isTailCall
9722 // because the return pointer needs to be adjusted by the size of
9723 // the copied memory.
9724 SDValue Root = getMemoryRoot();
9725 SDValue MC = DAG.getMemcpy(
9726 Root, sdl, Dst, Src, Size, DstAlign, SrcAlign, false, false,
9727 /*CI=*/nullptr, std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
9728 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
9729 assert(MC.getNode() != nullptr &&
9730 "** memcpy should not be lowered as TailCall in mempcpy context **");
9731 DAG.setRoot(MC);
9732
9733 // Check if Size needs to be truncated or extended.
9734 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
9735
9736 // Adjust return pointer to point just past the last dst byte.
9737 SDValue DstPlusSize = DAG.getMemBasePlusOffset(Dst, Size, sdl);
9738 setValue(&I, DstPlusSize);
9739 return true;
9740}
9741
9742/// See if we can lower a strcpy call into an optimized form. If so, return
9743/// true and lower it, otherwise return false and it will be lowered like a
9744/// normal call.
9745/// The caller already checked that \p I calls the appropriate LibFunc with a
9746/// correct prototype.
9747bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9748 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9749
9750 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9751 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcpy(
9752 DAG, getCurSDLoc(), getRoot(), getValue(Arg0), getValue(Arg1),
9753 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), isStpcpy, &I);
9754 if (Res.first.getNode()) {
9755 setValue(&I, Res.first);
9756 DAG.setRoot(Res.second);
9757 return true;
9758 }
9759
9760 return false;
9761}
9762
9763/// See if we can lower a strcmp call into an optimized form. If so, return
9764/// true and lower it, otherwise return false and it will be lowered like a
9765/// normal call.
9766/// The caller already checked that \p I calls the appropriate LibFunc with a
9767/// correct prototype.
9768bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9769 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9770
9771 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9772 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrcmp(
9773 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), getValue(Arg1),
9774 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), &I);
9775 if (Res.first.getNode()) {
9776 processIntegerCallValue(I, Res.first, true);
9777 PendingLoads.push_back(Res.second);
9778 return true;
9779 }
9780
9781 return false;
9782}
9783
9784/// See if we can lower a strlen call into an optimized form. If so, return
9785/// true and lower it, otherwise return false and it will be lowered like a
9786/// normal call.
9787/// The caller already checked that \p I calls the appropriate LibFunc with a
9788/// correct prototype.
9789bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9790 const Value *Arg0 = I.getArgOperand(0);
9791
9792 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9793 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrlen(
9794 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), &I);
9795 if (Res.first.getNode()) {
9796 processIntegerCallValue(I, Res.first, false);
9797 PendingLoads.push_back(Res.second);
9798 return true;
9799 }
9800
9801 return false;
9802}
9803
9804/// See if we can lower a strnlen call into an optimized form. If so, return
9805/// true and lower it, otherwise return false and it will be lowered like a
9806/// normal call.
9807/// The caller already checked that \p I calls the appropriate LibFunc with a
9808/// correct prototype.
9809bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9810 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9811
9812 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9813 std::pair<SDValue, SDValue> Res =
9814 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
9815 getValue(Arg0), getValue(Arg1),
9816 MachinePointerInfo(Arg0));
9817 if (Res.first.getNode()) {
9818 processIntegerCallValue(I, Res.first, false);
9819 PendingLoads.push_back(Res.second);
9820 return true;
9821 }
9822
9823 return false;
9824}
9825
9826/// See if we can lower a Strstr call into an optimized form. If so, return
9827/// true and lower it, otherwise return false and it will be lowered like a
9828/// normal call.
9829/// The caller already checked that \p I calls the appropriate LibFunc with a
9830/// correct prototype.
9831bool SelectionDAGBuilder::visitStrstrCall(const CallInst &I) {
9832 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
9833 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9834 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrstr(
9835 DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), getValue(Arg1), &I);
9836 if (Res.first) {
9837 processIntegerCallValue(I, Res.first, false);
9838 PendingLoads.push_back(Res.second);
9839 return true;
9840 }
9841 return false;
9842}
9843
9844/// See if we can lower a unary floating-point operation into an SDNode with
9845/// the specified Opcode. If so, return true and lower it, otherwise return
9846/// false and it will be lowered like a normal call.
9847/// The caller already checked that \p I calls the appropriate LibFunc with a
9848/// correct prototype.
9849bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9850 unsigned Opcode) {
9851 // We already checked this call's prototype; verify it doesn't modify errno.
9852 // Do not perform optimizations for call sites that require strict
9853 // floating-point semantics.
9854 if (!I.onlyReadsMemory() || I.isStrictFP())
9855 return false;
9856
9857 SDNodeFlags Flags;
9858 Flags.copyFMF(cast<FPMathOperator>(I));
9859
9860 SDValue Tmp = getValue(I.getArgOperand(0));
9861 setValue(&I,
9862 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
9863 return true;
9864}
9865
9866/// See if we can lower a binary floating-point operation into an SDNode with
9867/// the specified Opcode. If so, return true and lower it. Otherwise return
9868/// false, and it will be lowered like a normal call.
9869/// The caller already checked that \p I calls the appropriate LibFunc with a
9870/// correct prototype.
9871bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9872 unsigned Opcode) {
9873 // We already checked this call's prototype; verify it doesn't modify errno.
9874 // Do not perform optimizations for call sites that require strict
9875 // floating-point semantics.
9876 if (!I.onlyReadsMemory() || I.isStrictFP())
9877 return false;
9878
9879 SDNodeFlags Flags;
9880 Flags.copyFMF(cast<FPMathOperator>(I));
9881
9882 SDValue Tmp0 = getValue(I.getArgOperand(0));
9883 SDValue Tmp1 = getValue(I.getArgOperand(1));
9884 EVT VT = Tmp0.getValueType();
9885 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
9886 return true;
9887}
9888
9889void SelectionDAGBuilder::visitCall(const CallInst &I) {
9890 // Handle inline assembly differently.
9891 if (I.isInlineAsm()) {
9892 visitInlineAsm(I);
9893 return;
9894 }
9895
9897
9898 if (Function *F = I.getCalledFunction()) {
9899 if (F->isDeclaration()) {
9900 // Is this an LLVM intrinsic?
9901 if (unsigned IID = F->getIntrinsicID()) {
9902 visitIntrinsicCall(I, IID);
9903 return;
9904 }
9905 }
9906
9907 // Check for well-known libc/libm calls. If the function is internal, it
9908 // can't be a library call. Don't do the check if marked as nobuiltin for
9909 // some reason.
9910 // This code should not handle libcalls that are already canonicalized to
9911 // intrinsics by the middle-end.
9912 LibFunc Func = !I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName()
9913 ? LibInfo->getLibFunc(*F)
9914 : NotLibFunc;
9915 if (LibInfo->hasOptimizedCodeGen(Func)) {
9916 switch (Func) {
9917 default: break;
9918 case LibFunc_bcmp:
9919 if (visitMemCmpBCmpCall(I))
9920 return;
9921 break;
9922 case LibFunc_sin:
9923 case LibFunc_sinf:
9924 case LibFunc_sinl:
9925 if (visitUnaryFloatCall(I, ISD::FSIN))
9926 return;
9927 break;
9928 case LibFunc_cos:
9929 case LibFunc_cosf:
9930 case LibFunc_cosl:
9931 if (visitUnaryFloatCall(I, ISD::FCOS))
9932 return;
9933 break;
9934 case LibFunc_tan:
9935 case LibFunc_tanf:
9936 case LibFunc_tanl:
9937 if (visitUnaryFloatCall(I, ISD::FTAN))
9938 return;
9939 break;
9940 case LibFunc_asin:
9941 case LibFunc_asinf:
9942 case LibFunc_asinl:
9943 if (visitUnaryFloatCall(I, ISD::FASIN))
9944 return;
9945 break;
9946 case LibFunc_acos:
9947 case LibFunc_acosf:
9948 case LibFunc_acosl:
9949 if (visitUnaryFloatCall(I, ISD::FACOS))
9950 return;
9951 break;
9952 case LibFunc_atan:
9953 case LibFunc_atanf:
9954 case LibFunc_atanl:
9955 if (visitUnaryFloatCall(I, ISD::FATAN))
9956 return;
9957 break;
9958 case LibFunc_atan2:
9959 case LibFunc_atan2f:
9960 case LibFunc_atan2l:
9961 if (visitBinaryFloatCall(I, ISD::FATAN2))
9962 return;
9963 break;
9964 case LibFunc_sinh:
9965 case LibFunc_sinhf:
9966 case LibFunc_sinhl:
9967 if (visitUnaryFloatCall(I, ISD::FSINH))
9968 return;
9969 break;
9970 case LibFunc_cosh:
9971 case LibFunc_coshf:
9972 case LibFunc_coshl:
9973 if (visitUnaryFloatCall(I, ISD::FCOSH))
9974 return;
9975 break;
9976 case LibFunc_tanh:
9977 case LibFunc_tanhf:
9978 case LibFunc_tanhl:
9979 if (visitUnaryFloatCall(I, ISD::FTANH))
9980 return;
9981 break;
9982 case LibFunc_sqrt:
9983 case LibFunc_sqrtf:
9984 case LibFunc_sqrtl:
9985 case LibFunc_sqrt_finite:
9986 case LibFunc_sqrtf_finite:
9987 case LibFunc_sqrtl_finite:
9988 if (visitUnaryFloatCall(I, ISD::FSQRT))
9989 return;
9990 break;
9991 case LibFunc_log2:
9992 case LibFunc_log2f:
9993 case LibFunc_log2l:
9994 if (visitUnaryFloatCall(I, ISD::FLOG2))
9995 return;
9996 break;
9997 case LibFunc_exp2:
9998 case LibFunc_exp2f:
9999 case LibFunc_exp2l:
10000 if (visitUnaryFloatCall(I, ISD::FEXP2))
10001 return;
10002 break;
10003 case LibFunc_exp10:
10004 case LibFunc_exp10f:
10005 case LibFunc_exp10l:
10006 if (visitUnaryFloatCall(I, ISD::FEXP10))
10007 return;
10008 break;
10009 case LibFunc_strstr:
10010 if (visitStrstrCall(I))
10011 return;
10012 break;
10013 case LibFunc_memcmp:
10014 if (visitMemCmpBCmpCall(I))
10015 return;
10016 break;
10017 case LibFunc_memccpy:
10018 if (visitMemCCpyCall(I))
10019 return;
10020 break;
10021 case LibFunc_mempcpy:
10022 if (visitMemPCpyCall(I))
10023 return;
10024 break;
10025 case LibFunc_memchr:
10026 if (visitMemChrCall(I))
10027 return;
10028 break;
10029 case LibFunc_strcpy:
10030 if (visitStrCpyCall(I, false))
10031 return;
10032 break;
10033 case LibFunc_stpcpy:
10034 if (visitStrCpyCall(I, true))
10035 return;
10036 break;
10037 case LibFunc_strcmp:
10038 if (visitStrCmpCall(I))
10039 return;
10040 break;
10041 case LibFunc_strlen:
10042 if (visitStrLenCall(I))
10043 return;
10044 break;
10045 case LibFunc_strnlen:
10046 if (visitStrNLenCall(I))
10047 return;
10048 break;
10049 }
10050 }
10051 }
10052
10053 if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
10054 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
10055 return;
10056 }
10057
10058 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
10059 // have to do anything here to lower funclet bundles.
10060 // CFGuardTarget bundles are lowered in LowerCallTo.
10062 I, "calls",
10067
10068 SDValue Callee = getValue(I.getCalledOperand());
10069
10070 if (I.hasDeoptState())
10071 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
10072 else
10073 // Check if we can potentially perform a tail call. More detailed checking
10074 // is be done within LowerCallTo, after more information about the call is
10075 // known.
10076 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
10077}
10078
10080 const CallBase &CB, const BasicBlock *EHPadBB) {
10081 auto PAB = CB.getOperandBundle("ptrauth");
10082 const Value *CalleeV = CB.getCalledOperand();
10083
10084 // Gather the call ptrauth data from the operand bundle:
10085 // [ i32 <key>, i64 <discriminator> ]
10086 const auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
10087 const Value *Discriminator = PAB->Inputs[1];
10088
10089 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
10090 assert(Discriminator->getType()->isIntegerTy(64) &&
10091 "Invalid ptrauth discriminator");
10092
10093 // Look through ptrauth constants to find the raw callee.
10094 // Do a direct unauthenticated call if we found it and everything matches.
10095 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CalleeV))
10096 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
10097 DAG.getDataLayout()))
10098 return LowerCallTo(CB, getValue(CalleeCPA->getPointer()), CB.isTailCall(),
10099 CB.isMustTailCall(), EHPadBB);
10100
10101 // Functions should never be ptrauth-called directly.
10102 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
10103
10104 // Otherwise, do an authenticated indirect call.
10105 TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
10106 getValue(Discriminator)};
10107
10108 LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
10109 EHPadBB, &PAI);
10110}
10111
10112namespace {
10113
10114/// AsmOperandInfo - This contains information for each constraint that we are
10115/// lowering.
10116class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
10117public:
10118 /// CallOperand - If this is the result output operand or a clobber
10119 /// this is null, otherwise it is the incoming operand to the CallInst.
10120 /// This gets modified as the asm is processed.
10121 SDValue CallOperand;
10122
10123 /// AssignedRegs - If this is a register or register class operand, this
10124 /// contains the set of register corresponding to the operand.
10125 RegsForValue AssignedRegs;
10126
10127 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
10128 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
10129 }
10130
10131 /// Whether or not this operand accesses memory
10132 bool hasMemory(const TargetLowering &TLI) const {
10133 // Indirect operand accesses access memory.
10134 if (isIndirect)
10135 return true;
10136
10137 for (const auto &Code : Codes)
10138 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
10139 return true;
10140
10141 return false;
10142 }
10143};
10144
10145
10146} // end anonymous namespace
10147
10148/// Make sure that the output operand \p OpInfo and its corresponding input
10149/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
10150/// out).
10151static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
10152 SDISelAsmOperandInfo &MatchingOpInfo,
10153 SelectionDAG &DAG) {
10154 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
10155 return;
10156
10158 const auto &TLI = DAG.getTargetLoweringInfo();
10159
10160 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
10161 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
10162 OpInfo.ConstraintVT);
10163 std::pair<unsigned, const TargetRegisterClass *> InputRC =
10164 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
10165 MatchingOpInfo.ConstraintVT);
10166 const bool OutOpIsIntOrFP =
10167 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
10168 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
10169 MatchingOpInfo.ConstraintVT.isFloatingPoint();
10170 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
10171 // FIXME: error out in a more elegant fashion
10172 report_fatal_error("Unsupported asm: input constraint"
10173 " with a matching output constraint of"
10174 " incompatible type!");
10175 }
10176 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
10177}
10178
10179/// Get a direct memory input to behave well as an indirect operand.
10180/// This may introduce stores, hence the need for a \p Chain.
10181/// \return The (possibly updated) chain.
10182static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
10183 SDISelAsmOperandInfo &OpInfo,
10184 SelectionDAG &DAG) {
10186
10187 // If we don't have an indirect input, put it in the constpool if we can,
10188 // otherwise spill it to a stack slot.
10189 // TODO: This isn't quite right. We need to handle these according to
10190 // the addressing mode that the constraint wants. Also, this may take
10191 // an additional register for the computation and we don't want that
10192 // either.
10193
10194 // If the operand is a float, integer, or vector constant, spill to a
10195 // constant pool entry to get its address.
10196 const Value *OpVal = OpInfo.CallOperandVal;
10197 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
10199 OpInfo.CallOperand = DAG.getConstantPool(
10200 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
10201 return Chain;
10202 }
10203
10204 // Otherwise, create a stack slot and emit a store to it before the asm.
10205 Type *Ty = OpVal->getType();
10206 auto &DL = DAG.getDataLayout();
10207 TypeSize TySize = DL.getTypeAllocSize(Ty);
10210 int StackID = 0;
10211 if (TySize.isScalable())
10212 StackID = TFI->getStackIDForScalableVectors();
10213 int SSFI = MF.getFrameInfo().CreateStackObject(TySize.getKnownMinValue(),
10214 DL.getPrefTypeAlign(Ty), false,
10215 nullptr, StackID);
10216 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
10217 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
10219 TLI.getMemValueType(DL, Ty));
10220 OpInfo.CallOperand = StackSlot;
10221
10222 return Chain;
10223}
10224
10225/// GetRegistersForValue - Assign registers (virtual or physical) for the
10226/// specified operand. We prefer to assign virtual registers, to allow the
10227/// register allocator to handle the assignment process. However, if the asm
10228/// uses features that we can't model on machineinstrs, we have SDISel do the
10229/// allocation. This produces generally horrible, but correct, code.
10230///
10231/// OpInfo describes the operand
10232/// RefOpInfo describes the matching operand if any, the operand otherwise
10233static std::optional<unsigned>
10235 SDISelAsmOperandInfo &OpInfo,
10236 SDISelAsmOperandInfo &RefOpInfo) {
10237 LLVMContext &Context = *DAG.getContext();
10239
10243
10244 // No work to do for memory/address operands.
10245 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
10246 OpInfo.ConstraintType == TargetLowering::C_Address)
10247 return std::nullopt;
10248
10249 // If this is a constraint for a single physreg, or a constraint for a
10250 // register class, find it.
10251 unsigned AssignedReg;
10252 const TargetRegisterClass *RC;
10253 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
10254 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
10255 // RC is unset only on failure. Return immediately.
10256 if (!RC)
10257 return std::nullopt;
10258
10259 // Get the actual register value type. This is important, because the user
10260 // may have asked for (e.g.) the AX register in i32 type. We need to
10261 // remember that AX is actually i16 to get the right extension.
10262 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
10263
10264 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10265 // If this is an FP operand in an integer register (or visa versa), or more
10266 // generally if the operand value disagrees with the register class we plan
10267 // to stick it in, fix the operand type.
10268 //
10269 // If this is an input value, the bitcast to the new type is done now.
10270 // Bitcast for output value is done at the end of visitInlineAsm().
10271 if ((OpInfo.Type == InlineAsm::isOutput ||
10272 OpInfo.Type == InlineAsm::isInput) &&
10273 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
10274 // Try to convert to the first EVT that the reg class contains. If the
10275 // types are identical size, use a bitcast to convert (e.g. two differing
10276 // vector types). Note: output bitcast is done at the end of
10277 // visitInlineAsm().
10278 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10279 // Exclude indirect inputs while they are unsupported because the code
10280 // to perform the load is missing and thus OpInfo.CallOperand still
10281 // refers to the input address rather than the pointed-to value.
10282 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
10283 OpInfo.CallOperand =
10284 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
10285 OpInfo.ConstraintVT = RegVT;
10286 // If the operand is an FP value and we want it in integer registers,
10287 // use the corresponding integer type. This turns an f64 value into
10288 // i64, which can be passed with two i32 values on a 32-bit machine.
10289 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10290 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
10291 // No integer type of that size to bitcast to (e.g. f80).
10292 if (!VT.isValid())
10293 return std::nullopt;
10294 if (OpInfo.Type == InlineAsm::isInput)
10295 OpInfo.CallOperand =
10296 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
10297 OpInfo.ConstraintVT = VT;
10298 }
10299 }
10300 }
10301
10302 // No need to allocate a matching input constraint since the constraint it's
10303 // matching to has already been allocated.
10304 if (OpInfo.isMatchingInputConstraint())
10305 return std::nullopt;
10306
10307 EVT ValueVT = OpInfo.ConstraintVT;
10308 if (OpInfo.ConstraintVT == MVT::Other)
10309 ValueVT = RegVT;
10310
10311 // Initialize NumRegs.
10312 unsigned NumRegs = 1;
10313 if (OpInfo.ConstraintVT != MVT::Other)
10314 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
10315
10316 // If this is a constraint for a specific physical register, like {r17},
10317 // assign it now.
10318
10319 // If this associated to a specific register, initialize iterator to correct
10320 // place. If virtual, make sure we have enough registers
10321
10322 // Initialize iterator if necessary
10325
10326 // Do not check for single registers.
10327 if (AssignedReg) {
10328 I = std::find(I, RC->end(), AssignedReg);
10329 if (I == RC->end()) {
10330 // RC does not contain the selected register, which indicates a
10331 // mismatch between the register and the required type/bitwidth.
10332 return {AssignedReg};
10333 }
10334 }
10335
10336 for (; NumRegs; --NumRegs, ++I) {
10337 assert(I != RC->end() && "Ran out of registers to allocate!");
10338 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
10339 Regs.push_back(R);
10340 }
10341
10342 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
10343 return std::nullopt;
10344}
10345
10346static unsigned
10348 const std::vector<SDValue> &AsmNodeOperands) {
10349 // Scan until we find the definition we already emitted of this operand.
10350 unsigned CurOp = InlineAsm::Op_FirstOperand;
10351 for (; OperandNo; --OperandNo) {
10352 // Advance to the next operand.
10353 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10354 const InlineAsm::Flag F(OpFlag);
10355 assert(
10356 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
10357 "Skipped past definitions?");
10358 CurOp += F.getNumOperandRegisters() + 1;
10359 }
10360 return CurOp;
10361}
10362
10363namespace {
10364
10365class ExtraFlags {
10366 unsigned Flags = 0;
10367
10368public:
10369 explicit ExtraFlags(const CallBase &Call) {
10370 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
10371 if (IA->hasSideEffects())
10373 if (IA->isAlignStack())
10375 if (IA->canThrow())
10377 if (Call.isConvergent())
10379 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
10380 }
10381
10382 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
10383 // Ideally, we would only check against memory constraints. However, the
10384 // meaning of an Other constraint can be target-specific and we can't easily
10385 // reason about it. Therefore, be conservative and set MayLoad/MayStore
10386 // for Other constraints as well.
10389 if (OpInfo.Type == InlineAsm::isInput)
10391 else if (OpInfo.Type == InlineAsm::isOutput)
10393 else if (OpInfo.Type == InlineAsm::isClobber)
10395 }
10396 }
10397
10398 unsigned get() const { return Flags; }
10399};
10400
10401} // end anonymous namespace
10402
10403static bool isFunction(SDValue Op) {
10404 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
10405 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
10406 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
10407
10408 // In normal "call dllimport func" instruction (non-inlineasm) it force
10409 // indirect access by specifing call opcode. And usually specially print
10410 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
10411 // not do in this way now. (In fact, this is similar with "Data Access"
10412 // action). So here we ignore dllimport function.
10413 if (Fn && !Fn->hasDLLImportStorageClass())
10414 return true;
10415 }
10416 }
10417 return false;
10418}
10419
10420namespace {
10421
10422struct ConstraintDecisionInfo {
10423 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
10424 std::vector<SDValue> AsmNodeOperands;
10425 SDValue Glue, Chain;
10426 bool HasSideEffect = false;
10427 MCSymbol *BeginLabel = nullptr;
10428
10429 SmallVector<char> Buffer;
10430 raw_svector_ostream ErrorMsg;
10431
10432 ConstraintDecisionInfo() : ErrorMsg(Buffer) {}
10433};
10434
10435} // end anonymous namespace
10436
10437/// Construct operand info objects.
10438static bool
10439constructOperandInfo(ConstraintDecisionInfo &Info,
10440 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10441 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10442 ExtraFlags &ExtraInfo) {
10443 for (auto &T : TargetConstraints) {
10444 Info.ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
10445 SDISelAsmOperandInfo &OpInfo = Info.ConstraintOperands.back();
10446
10447 if (OpInfo.CallOperandVal)
10448 OpInfo.CallOperand = Builder.getValue(OpInfo.CallOperandVal);
10449
10450 if (!Info.HasSideEffect)
10451 Info.HasSideEffect = OpInfo.hasMemory(TLI);
10452
10453 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
10454 // FIXME: Could we compute this on OpInfo rather than T?
10455
10456 // Compute the constraint code and ConstraintType to use.
10457 TLI.ComputeConstraintToUse(T, SDValue());
10458
10459 if (T.ConstraintType == TargetLowering::C_Immediate && OpInfo.CallOperand &&
10460 !isa<ConstantSDNode>(OpInfo.CallOperand)) {
10461 // We've delayed emitting a diagnostic like the "n" constraint because
10462 // inlining could cause an integer showing up.
10463 Info.ErrorMsg << "constraint '" << T.ConstraintCode
10464 << "' expects an integer constant expression";
10465 return true;
10466 }
10467
10468 ExtraInfo.update(T);
10469 }
10470
10471 return false;
10472}
10473
10474/// Compute which constraint option to use for each operand.
10475static void
10476computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
10477 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10478 SelectionDAGBuilder &Builder, const TargetLowering &TLI,
10479 const TargetMachine &TM, SelectionDAG &DAG) {
10480 const auto *IA = cast<InlineAsm>(Call.getCalledOperand());
10482 IA->collectAsmStrs(AsmStrs);
10483
10484 int OpNo = -1;
10485 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10486 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10487 OpNo++;
10488
10489 // If this is an output operand with a matching input operand, look up the
10490 // matching input. If their types mismatch, e.g. one is an integer, the
10491 // other is floating point, or their sizes are different, flag it as an
10492 // error.
10493 if (OpInfo.hasMatchingInput()) {
10494 SDISelAsmOperandInfo &Input =
10495 Info.ConstraintOperands[OpInfo.MatchingInput];
10496 patchMatchingInput(OpInfo, Input, DAG);
10497 }
10498
10499 // Compute the constraint code and ConstraintType to use.
10500 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
10501
10502 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10503 OpInfo.Type == InlineAsm::isClobber) ||
10504 OpInfo.ConstraintType == TargetLowering::C_Address)
10505 continue;
10506
10507 // In Linux PIC model, there are 4 cases about value/label addressing:
10508 //
10509 // 1: Function call or Label jmp inside the module.
10510 // 2: Data access (such as global variable, static variable) inside module.
10511 // 3: Function call or Label jmp outside the module.
10512 // 4: Data access (such as global variable) outside the module.
10513 //
10514 // Due to current llvm inline asm architecture designed to not "recognize"
10515 // the asm code, there are quite troubles for us to treat mem addressing
10516 // differently for same value/adress used in different instructions.
10517 // For example, in pic model, call a func may in plt way or direclty
10518 // pc-related, but lea/mov a function adress may use got.
10519 //
10520 // Here we try to "recognize" function call for the case 1 and case 3 in
10521 // inline asm. And try to adjust the constraint for them.
10522 //
10523 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10524 // label, so here we don't handle jmp function label now, but we need to
10525 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10526 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
10527 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10529 OpInfo.isIndirect = false;
10530 OpInfo.ConstraintType = TargetLowering::C_Address;
10531 }
10532
10533 // If this is a memory input, and if the operand is not indirect, do what we
10534 // need to provide an address for the memory input.
10535 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10536 !OpInfo.isIndirect) {
10537 assert((OpInfo.isMultipleAlternative ||
10538 (OpInfo.Type == InlineAsm::isInput)) &&
10539 "Can only indirectify direct input operands!");
10540
10541 // Memory operands really want the address of the value.
10542 Info.Chain = getAddressForMemoryInput(Info.Chain, Builder.getCurSDLoc(),
10543 OpInfo, DAG);
10544
10545 // There is no longer a Value* corresponding to this operand.
10546 OpInfo.CallOperandVal = nullptr;
10547
10548 // It is now an indirect operand.
10549 OpInfo.isIndirect = true;
10550 }
10551 }
10552}
10553
10554/// Prepare DAG-level operands. As part of this, assign virtual and physical
10555/// registers for inputs and output.
10556static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info,
10557 const CallBase &Call,
10558 SelectionDAGBuilder &Builder,
10559 const TargetLowering &TLI,
10560 SelectionDAG &DAG) {
10561 SDLoc DL = Builder.getCurSDLoc();
10562 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10563 // Assign Registers.
10564 SDISelAsmOperandInfo &RefOpInfo =
10565 OpInfo.isMatchingInputConstraint()
10566 ? Info.ConstraintOperands[OpInfo.getMatchedOperand()]
10567 : OpInfo;
10568 const auto RegError = getRegistersForValue(DAG, DL, OpInfo, RefOpInfo);
10569 if (RegError) {
10570 const MachineFunction &MF = DAG.getMachineFunction();
10572 const char *RegName = TRI.getName(*RegError);
10573 Info.ErrorMsg << "register '" << RegName << "' allocated for constraint '"
10574 << OpInfo.ConstraintCode
10575 << "' does not match required type";
10576 return true;
10577 }
10578
10579 auto DetectWriteToReservedRegister = [&]() {
10580 const MachineFunction &MF = DAG.getMachineFunction();
10582
10583 for (Register Reg : OpInfo.AssignedRegs.Regs) {
10584 if (Reg.isPhysical() && TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
10585 Info.ErrorMsg << "write to reserved register '"
10586 << TRI.getRegAsmName(Reg) << "'";
10587 return true;
10588 }
10589 }
10590
10591 return false;
10592 };
10593 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10594 (OpInfo.Type == InlineAsm::isInput &&
10595 !OpInfo.isMatchingInputConstraint())) &&
10596 "Only address as input operand is allowed.");
10597
10598 switch (OpInfo.Type) {
10600 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10601 const InlineAsm::ConstraintCode ConstraintID =
10602 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10604 "Failed to convert memory constraint code to constraint id.");
10605
10606 // Add information to the INLINEASM node to know about this output.
10608 OpFlags.setMemConstraint(ConstraintID);
10609 Info.AsmNodeOperands.push_back(
10610 DAG.getTargetConstant(OpFlags, DL, MVT::i32));
10611 Info.AsmNodeOperands.push_back(OpInfo.CallOperand);
10612 } else {
10613 // Otherwise, this outputs to a register (directly for C_Register /
10614 // C_RegisterClass, and a target-defined fashion for
10615 // C_Immediate/C_Other). Find a register that we can use.
10616 if (OpInfo.AssignedRegs.Regs.empty()) {
10617 Info.ErrorMsg << "could not allocate output register for "
10618 << "constraint '" << OpInfo.ConstraintCode << "'";
10619 return true;
10620 }
10621
10622 if (DetectWriteToReservedRegister())
10623 return true;
10624
10625 // Add information to the INLINEASM node to know that this register is
10626 // set.
10627 OpInfo.AssignedRegs.AddInlineAsmOperands(
10628 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10630 false, 0, DL, DAG, Info.AsmNodeOperands);
10631 }
10632 break;
10633
10634 case InlineAsm::isInput:
10635 case InlineAsm::isLabel: {
10636 SDValue InOperandVal = OpInfo.CallOperand;
10637
10638 if (OpInfo.isMatchingInputConstraint()) {
10639 // If this is required to match an output register we have already set,
10640 // just use its register.
10641 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
10642 Info.AsmNodeOperands);
10643 InlineAsm::Flag Flag(Info.AsmNodeOperands[CurOp]->getAsZExtVal());
10644 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10645 if (OpInfo.isIndirect) {
10646 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10647 Info.ErrorMsg << "inline asm not supported yet: cannot handle "
10648 << "tied indirect register inputs";
10649 return true;
10650 }
10651
10656 auto *R = cast<RegisterSDNode>(Info.AsmNodeOperands[CurOp + 1]);
10657 Register TiedReg = R->getReg();
10658 MVT RegVT = R->getSimpleValueType(0);
10659 const TargetRegisterClass *RC;
10660 if (TiedReg.isVirtual()) {
10661 RC = MRI.getRegClass(TiedReg);
10662 } else if (RegVT == MVT::Untyped) {
10663 RC = TRI.getMinimalPhysRegClass(TiedReg);
10664 } else {
10665 RC = TLI.getRegClassFor(RegVT);
10666 if (!RC->contains(TiedReg)) {
10667 const TargetRegisterClass *PhysRC =
10668 TRI.getMinimalPhysRegClass(TiedReg);
10669 if (PhysRC->isAllocatable())
10670 RC = PhysRC;
10671 }
10672 }
10673
10674 for (unsigned I = 0, E = Flag.getNumOperandRegisters(); I != E; ++I)
10675 Regs.push_back(MRI.createVirtualRegister(RC));
10676
10677 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10678
10679 // Use the produced MatchedRegs object to
10680 MatchedRegs.getCopyToRegs(InOperandVal, DAG, DL, Info.Chain,
10681 &Info.Glue, &Call);
10683 OpInfo.getMatchedOperand(), DL, DAG,
10684 Info.AsmNodeOperands);
10685 break;
10686 }
10687
10688 assert(Flag.isMemKind() && "Unknown matching constraint!");
10689 assert(Flag.getNumOperandRegisters() == 1 &&
10690 "Unexpected number of operands");
10691
10692 // Add information to the INLINEASM node to know about this input.
10693 // See InlineAsm.h isUseOperandTiedToDef.
10694 Flag.clearMemConstraint();
10695 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10696 Info.AsmNodeOperands.push_back(DAG.getTargetConstant(
10697 Flag, DL, TLI.getPointerTy(DAG.getDataLayout())));
10698 Info.AsmNodeOperands.push_back(Info.AsmNodeOperands[CurOp + 1]);
10699 break;
10700 }
10701
10702 // Treat indirect 'X' constraint as memory.
10703 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10704 OpInfo.isIndirect)
10705 OpInfo.ConstraintType = TargetLowering::C_Memory;
10706
10707 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10708 OpInfo.ConstraintType == TargetLowering::C_Other) {
10709 std::vector<SDValue> Ops;
10710 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
10711 Ops, DAG);
10712 if (Ops.empty()) {
10713 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10714 if (isa<ConstantSDNode>(InOperandVal)) {
10715 Info.ErrorMsg << "value out of range for constraint '"
10716 << OpInfo.ConstraintCode << "'";
10717 return true;
10718 }
10719
10720 Info.ErrorMsg << "invalid operand for inline asm constraint '"
10721 << OpInfo.ConstraintCode << "'";
10722 return true;
10723 }
10724
10725 // Add information to the INLINEASM node to know about this input.
10726 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10727 Info.AsmNodeOperands.push_back(DAG.getTargetConstant(
10728 ResOpType, DL, TLI.getPointerTy(DAG.getDataLayout())));
10729 llvm::append_range(Info.AsmNodeOperands, Ops);
10730 break;
10731 }
10732
10733 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10734 assert((OpInfo.isIndirect ||
10735 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10736 "Operand must be indirect to be a mem!");
10737
10738 assert(InOperandVal.getValueType() ==
10739 TLI.getPointerTy(
10740 DAG.getDataLayout(),
10741 OpInfo.CallOperandVal
10742 ? cast<PointerType>(OpInfo.CallOperandVal->getType())
10743 ->getAddressSpace()
10744 : 0) &&
10745 "Memory operands expect pointer values");
10746
10747 const InlineAsm::ConstraintCode ConstraintID =
10748 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10750 "Failed to convert memory constraint code to constraint id.");
10751
10752 // Add information to the INLINEASM node to know about this input.
10754 ResOpType.setMemConstraint(ConstraintID);
10755 Info.AsmNodeOperands.push_back(
10756 DAG.getTargetConstant(ResOpType, DL, MVT::i32));
10757 Info.AsmNodeOperands.push_back(InOperandVal);
10758 break;
10759 }
10760
10761 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10762 const InlineAsm::ConstraintCode ConstraintID =
10763 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10765 "Failed to convert memory constraint code to constraint id.");
10766
10768
10769 SDValue AsmOp = InOperandVal;
10770 if (TLI.supportsInlineAsmFuncOperands() && isFunction(InOperandVal)) {
10771 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
10772 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10773 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
10774 InOperandVal.getValueType(),
10775 GA->getOffset());
10776 }
10777
10778 // Add information to the INLINEASM node to know about this input.
10779 ResOpType.setMemConstraint(ConstraintID);
10780
10781 Info.AsmNodeOperands.push_back(
10782 DAG.getTargetConstant(ResOpType, DL, MVT::i32));
10783 Info.AsmNodeOperands.push_back(AsmOp);
10784 break;
10785 }
10786
10787 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10788 OpInfo.ConstraintType != TargetLowering::C_Register) {
10789 Info.ErrorMsg << "unknown asm constraint '" << OpInfo.ConstraintCode
10790 << "'";
10791 return true;
10792 }
10793
10794 // TODO: Support this.
10795 if (OpInfo.isIndirect) {
10796 Info.ErrorMsg << "cannot handle indirect register inputs yet for "
10797 << "constraint '" << OpInfo.ConstraintCode << "'";
10798 return true;
10799 }
10800
10801 // Copy the input into the appropriate registers.
10802 if (OpInfo.AssignedRegs.Regs.empty()) {
10803 Info.ErrorMsg << "could not allocate input reg for constraint '"
10804 << OpInfo.ConstraintCode << "'";
10805 return true;
10806 }
10807
10808 if (DetectWriteToReservedRegister())
10809 return true;
10810
10811 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, DL, Info.Chain,
10812 &Info.Glue, &Call);
10813 OpInfo.AssignedRegs.AddInlineAsmOperands(
10814 InlineAsm::Kind::RegUse, false, 0, DL, DAG, Info.AsmNodeOperands);
10815 break;
10816 }
10817
10819 // Add the clobbered value to the operand list, so that the register
10820 // allocator is aware that the physreg got clobbered.
10821 if (!OpInfo.AssignedRegs.Regs.empty())
10822 OpInfo.AssignedRegs.AddInlineAsmOperands(
10823 InlineAsm::Kind::Clobber, false, 0, DL, DAG, Info.AsmNodeOperands);
10824 break;
10825 }
10826 }
10827
10828 return false;
10829}
10830
10831/// DetermineConstraints - Find the constraints to use for inline asm operands.
10832static bool
10833determineConstraints(ConstraintDecisionInfo &Info,
10834 TargetLowering::AsmOperandInfoVector &TargetConstraints,
10835 const CallBase &Call, SelectionDAGBuilder &Builder,
10836 const TargetLowering &TLI, const TargetMachine &TM,
10837 SelectionDAG &DAG, const BasicBlock *EHPadBB) {
10838 const auto *IA = cast<InlineAsm>(Call.getCalledOperand());
10839 ExtraFlags ExtraInfo(Call);
10840
10841 // First pass: Construct operand info objects.
10842 Info.HasSideEffect = IA->hasSideEffects();
10843 if (constructOperandInfo(Info, TargetConstraints, Builder, TLI, ExtraInfo))
10844 return true;
10845
10846 // We won't need to flush pending loads if this asm doesn't touch
10847 // memory and is nonvolatile.
10848 Info.Chain = Info.HasSideEffect ? Builder.getRoot() : DAG.getRoot();
10849
10850 bool IsCallBr = isa<CallBrInst>(Call);
10851 bool EmitEHLabels = isa<InvokeInst>(Call);
10852 if (IsCallBr || EmitEHLabels)
10853 // If this is a callbr or invoke we need to flush pending exports since
10854 // inlineasm_br and invoke are terminators.
10855 // We need to do this before nodes are glued to the inlineasm_br node.
10856 Info.Chain = Builder.getControlRoot();
10857
10858 if (EmitEHLabels)
10859 Info.Chain = Builder.lowerStartEH(Info.Chain, EHPadBB, Info.BeginLabel);
10860
10861 // Second pass: Compute which constraint option to use.
10862 computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM, DAG);
10863
10864 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10865 Info.AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
10866 Info.AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
10867 IA->getAsmString().data(), TLI.getProgramPointerTy(DAG.getDataLayout())));
10868
10869 // If we have a !srcloc metadata node associated with it, we want to attach
10870 // this to the ultimately generated inline asm machineinstr. To do this, we
10871 // pass in the third operand as this (potentially null) inline asm MDNode.
10872 const MDNode *SrcLoc = Call.getMetadata("srcloc");
10873 Info.AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
10874
10875 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10876 // bits as operand 3.
10877 Info.AsmNodeOperands.push_back(
10878 DAG.getTargetConstant(ExtraInfo.get(), Builder.getCurSDLoc(),
10879 TLI.getPointerTy(DAG.getDataLayout())));
10880
10881 // Third pass: Prepare DAG-level operands
10882 return prepareDAGLevelOperands(Info, Call, Builder, TLI, DAG);
10883}
10884
10885/// visitInlineAsm - Handle a call to an InlineAsm object.
10886void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
10887 const BasicBlock *EHPadBB) {
10888 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10890 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);
10891
10892 assert((!isa<InvokeInst>(Call) || EHPadBB) &&
10893 "InvokeInst must have an EHPadBB");
10894
10895 ConstraintDecisionInfo Info;
10896 if (determineConstraints(Info, TargetConstraints, Call, *this, TLI, TM, DAG,
10897 EHPadBB))
10898 return emitInlineAsmError(Call, Info.ErrorMsg.str());
10899
10900 SDValue Glue = Info.Glue;
10901 SDValue Chain = Info.Chain;
10902
10903 // Finish up input operands. Set the input chain and add the flag last.
10904 Info.AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10905 if (Glue.getNode())
10906 Info.AsmNodeOperands.push_back(Glue);
10907
10908 bool IsCallBr = isa<CallBrInst>(Call);
10909 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10910 Chain =
10911 DAG.getNode(ISDOpc, getCurSDLoc(), DAG.getVTList(MVT::Other, MVT::Glue),
10912 Info.AsmNodeOperands);
10913 Glue = Chain.getValue(1);
10914
10915 // Do additional work to generate outputs.
10916
10917 SmallVector<EVT, 1> ResultVTs;
10918 SmallVector<SDValue, 1> ResultValues;
10919 SmallVector<SDValue, 8> OutChains;
10920
10921 llvm::Type *CallResultType = Call.getType();
10922 ArrayRef<Type *> ResultTypes;
10923 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
10924 ResultTypes = StructResult->elements();
10925 else if (!CallResultType->isVoidTy())
10926 ResultTypes = ArrayRef(CallResultType);
10927
10928 auto CurResultType = ResultTypes.begin();
10929 auto handleRegAssign = [&](SDValue V) {
10930 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10931 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10932 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
10933 ++CurResultType;
10934 // If the type of the inline asm call site return value is different but has
10935 // same size as the type of the asm output bitcast it. One example of this
10936 // is for vectors with different width / number of elements. This can
10937 // happen for register classes that can contain multiple different value
10938 // types. The preg or vreg allocated may not have the same VT as was
10939 // expected.
10940 //
10941 // This can also happen for a return value that disagrees with the register
10942 // class it is put in, eg. a double in a general-purpose register on a
10943 // 32-bit machine.
10944 if (ResultVT != V.getValueType() &&
10945 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10946 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
10947 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10948 V.getValueType().isInteger()) {
10949 // If a result value was tied to an input value, the computed result
10950 // may have a wider width than the expected result. Extract the
10951 // relevant portion.
10952 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
10953 }
10954 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10955 ResultVTs.push_back(ResultVT);
10956 ResultValues.push_back(V);
10957 };
10958
10959 // Deal with output operands.
10960 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10961 if (OpInfo.Type == InlineAsm::isOutput) {
10962 SDValue Val;
10963 // Skip trivial output operands.
10964 if (OpInfo.AssignedRegs.Regs.empty())
10965 continue;
10966
10967 switch (OpInfo.ConstraintType) {
10970 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
10971 Chain, &Glue, &Call);
10972 break;
10975 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
10976 OpInfo, DAG);
10977 break;
10979 break; // Already handled.
10981 break; // Silence warning.
10983 assert(false && "Unexpected unknown constraint");
10984 }
10985
10986 // Indirect output manifest as stores. Record output chains.
10987 if (OpInfo.isIndirect) {
10988 const Value *Ptr = OpInfo.CallOperandVal;
10989 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10990 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
10991 MachinePointerInfo(Ptr));
10992 OutChains.push_back(Store);
10993 } else {
10994 // generate CopyFromRegs to associated registers.
10995 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10996 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10997 for (const SDValue &V : Val->op_values())
10998 handleRegAssign(V);
10999 } else
11000 handleRegAssign(Val);
11001 }
11002 }
11003 }
11004
11005 // Set results.
11006 if (!ResultValues.empty()) {
11007 assert(CurResultType == ResultTypes.end() &&
11008 "Mismatch in number of ResultTypes");
11009 assert(ResultValues.size() == ResultTypes.size() &&
11010 "Mismatch in number of output operands in asm result");
11011
11013 DAG.getVTList(ResultVTs), ResultValues);
11014 setValue(&Call, V);
11015 }
11016
11017 // Collect store chains.
11018 if (!OutChains.empty())
11019 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
11020
11021 if (const auto *II = dyn_cast<InvokeInst>(&Call))
11022 Chain = lowerEndEH(Chain, II, EHPadBB, Info.BeginLabel);
11023
11024 // Only Update Root if inline assembly has a memory effect.
11025 if (ResultValues.empty() || Info.HasSideEffect || !OutChains.empty() ||
11026 IsCallBr || isa<InvokeInst>(Call))
11027 DAG.setRoot(Chain);
11028}
11029
11030void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
11031 const Twine &Message) {
11032 LLVMContext &Ctx = *DAG.getContext();
11033 Ctx.diagnose(DiagnosticInfoInlineAsm(Call, Message));
11034
11035 // Make sure we leave the DAG in a valid state
11036 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11037 SmallVector<EVT, 1> ValueVTs;
11038 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
11039
11040 if (ValueVTs.empty())
11041 return;
11042
11044 for (const EVT &VT : ValueVTs)
11045 Ops.push_back(DAG.getUNDEF(VT));
11046
11047 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
11048}
11049
11050void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
11051 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
11052 MVT::Other, getRoot(),
11053 getValue(I.getArgOperand(0)),
11054 DAG.getSrcValue(I.getArgOperand(0))));
11055}
11056
11057void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
11058 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11059 const DataLayout &DL = DAG.getDataLayout();
11060 SDValue V = DAG.getVAArg(
11061 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
11062 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
11063 DL.getABITypeAlign(I.getType()).value());
11064 DAG.setRoot(V.getValue(1));
11065
11066 if (I.getType()->isPointerTy())
11067 V = DAG.getPtrExtOrTrunc(
11068 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
11069 setValue(&I, V);
11070}
11071
11072void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
11073 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
11074 MVT::Other, getRoot(),
11075 getValue(I.getArgOperand(0)),
11076 DAG.getSrcValue(I.getArgOperand(0))));
11077}
11078
11079void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
11080 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
11081 MVT::Other, getRoot(),
11082 getValue(I.getArgOperand(0)),
11083 getValue(I.getArgOperand(1)),
11084 DAG.getSrcValue(I.getArgOperand(0)),
11085 DAG.getSrcValue(I.getArgOperand(1))));
11086}
11087
11089 const Instruction &I,
11090 SDValue Op) {
11091 std::optional<ConstantRange> CR = getRange(I);
11092
11093 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
11094 return Op;
11095
11096 APInt Hi = CR->getUnsignedMax();
11097 unsigned Bits = std::max(Hi.getActiveBits(),
11098 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
11099
11100 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
11101
11102 SDLoc SL = getCurSDLoc();
11103
11104 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
11105 DAG.getValueType(SmallVT));
11106 unsigned NumVals = Op.getNode()->getNumValues();
11107 if (NumVals == 1)
11108 return ZExt;
11109
11111
11112 Ops.push_back(ZExt);
11113 for (unsigned I = 1; I != NumVals; ++I)
11114 Ops.push_back(Op.getValue(I));
11115
11116 return DAG.getMergeValues(Ops, SL);
11117}
11118
11120 SelectionDAG &DAG, const Instruction &I, SDValue Op) {
11121 FPClassTest Classes = getNoFPClass(I);
11122 if (Classes == fcNone)
11123 return Op;
11124
11125 SDLoc SL = getCurSDLoc();
11126 SDValue TestConst = DAG.getTargetConstant(Classes, SDLoc(), MVT::i32);
11127
11128 if (Op.getOpcode() != ISD::MERGE_VALUES) {
11129 return DAG.getNode(ISD::AssertNoFPClass, SL, Op.getValueType(), Op,
11130 TestConst);
11131 }
11132
11133 SmallVector<SDValue, 8> Ops(Op.getNumOperands());
11134 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
11135 SDValue MergeOp = Op.getOperand(I);
11136 Ops[I] = DAG.getNode(ISD::AssertNoFPClass, SL, MergeOp.getValueType(),
11137 MergeOp, TestConst);
11138 }
11139
11140 return DAG.getMergeValues(Ops, SL);
11141}
11142
11143/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
11144/// the call being lowered.
11145///
11146/// This is a helper for lowering intrinsics that follow a target calling
11147/// convention or require stack pointer adjustment. Only a subset of the
11148/// intrinsic's operands need to participate in the calling convention.
11151 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
11152 AttributeSet RetAttrs, bool IsPatchPoint) {
11154 Args.reserve(NumArgs);
11155
11156 // Populate the argument list.
11157 // Attributes for args start at offset 1, after the return attribute.
11158 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
11159 ArgI != ArgE; ++ArgI) {
11160 const Value *V = Call->getOperand(ArgI);
11161
11162 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
11163
11164 TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
11165 Entry.setAttributes(Call, ArgI);
11166 Args.push_back(Entry);
11167 }
11168
11170 .setChain(getRoot())
11171 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
11172 RetAttrs)
11173 .setDiscardResult(Call->use_empty())
11174 .setIsPatchPoint(IsPatchPoint)
11176 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
11177}
11178
11179/// Add a stack map intrinsic call's live variable operands to a stackmap
11180/// or patchpoint target node's operand list.
11181///
11182/// Constants are converted to TargetConstants purely as an optimization to
11183/// avoid constant materialization and register allocation.
11184///
11185/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
11186/// generate addess computation nodes, and so FinalizeISel can convert the
11187/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
11188/// address materialization and register allocation, but may also be required
11189/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
11190/// alloca in the entry block, then the runtime may assume that the alloca's
11191/// StackMap location can be read immediately after compilation and that the
11192/// location is valid at any point during execution (this is similar to the
11193/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
11194/// only available in a register, then the runtime would need to trap when
11195/// execution reaches the StackMap in order to read the alloca's location.
11196static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
11198 SelectionDAGBuilder &Builder) {
11199 SelectionDAG &DAG = Builder.DAG;
11200 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
11201 SDValue Op = Builder.getValue(Call.getArgOperand(I));
11202
11203 // Things on the stack are pointer-typed, meaning that they are already
11204 // legal and can be emitted directly to target nodes.
11206 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
11207 } else {
11208 // Otherwise emit a target independent node to be legalised.
11209 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
11210 }
11211 }
11212}
11213
11214/// Lower llvm.experimental.stackmap.
11215void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
11216 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
11217 // [live variables...])
11218
11219 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
11220
11221 SDValue Chain, InGlue, Callee;
11223
11224 SDLoc DL = getCurSDLoc();
11226
11227 // The stackmap intrinsic only records the live variables (the arguments
11228 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
11229 // intrinsic, this won't be lowered to a function call. This means we don't
11230 // have to worry about calling conventions and target specific lowering code.
11231 // Instead we perform the call lowering right here.
11232 //
11233 // chain, flag = CALLSEQ_START(chain, 0, 0)
11234 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
11235 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
11236 //
11237 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
11238 InGlue = Chain.getValue(1);
11239
11240 // Add the STACKMAP operands, starting with DAG house-keeping.
11241 Ops.push_back(Chain);
11242 Ops.push_back(InGlue);
11243
11244 // Add the <id>, <numShadowBytes> operands.
11245 //
11246 // These do not require legalisation, and can be emitted directly to target
11247 // constant nodes.
11249 assert(ID.getValueType() == MVT::i64);
11250 SDValue IDConst =
11251 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
11252 Ops.push_back(IDConst);
11253
11254 SDValue Shad = getValue(CI.getArgOperand(1));
11255 assert(Shad.getValueType() == MVT::i32);
11256 SDValue ShadConst =
11257 DAG.getTargetConstant(Shad->getAsZExtVal(), DL, Shad.getValueType());
11258 Ops.push_back(ShadConst);
11259
11260 // Add the live variables.
11261 addStackMapLiveVars(CI, 2, DL, Ops, *this);
11262
11263 // Create the STACKMAP node.
11264 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
11265 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
11266 InGlue = Chain.getValue(1);
11267
11268 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
11269
11270 // Stackmaps don't generate values, so nothing goes into the NodeMap.
11271
11272 // Set the root to the target-lowered call chain.
11273 DAG.setRoot(Chain);
11274
11275 // Inform the Frame Information that we have a stackmap in this function.
11276 FuncInfo.MF->getFrameInfo().setHasStackMap();
11277}
11278
11279/// Lower llvm.experimental.patchpoint directly to its target opcode.
11280void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
11281 const BasicBlock *EHPadBB) {
11282 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
11283 // i32 <numBytes>,
11284 // i8* <target>,
11285 // i32 <numArgs>,
11286 // [Args...],
11287 // [live variables...])
11288
11290 bool IsAnyRegCC = CC == CallingConv::AnyReg;
11291 bool HasDef = !CB.getType()->isVoidTy();
11292 SDLoc dl = getCurSDLoc();
11294
11295 // Handle immediate and symbolic callees.
11296 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
11297 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
11298 /*isTarget=*/true);
11299 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
11300 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
11301 SDLoc(SymbolicCallee),
11302 SymbolicCallee->getValueType(0));
11303
11304 // Get the real number of arguments participating in the call <numArgs>
11306 unsigned NumArgs = NArgVal->getAsZExtVal();
11307
11308 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
11309 // Intrinsics include all meta-operands up to but not including CC.
11310 unsigned NumMetaOpers = PatchPointOpers::CCPos;
11311 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
11312 "Not enough arguments provided to the patchpoint intrinsic");
11313
11314 // For AnyRegCC the arguments are lowered later on manually.
11315 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11316 Type *ReturnTy =
11317 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
11318
11319 TargetLowering::CallLoweringInfo CLI(DAG);
11320 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
11321 ReturnTy, CB.getAttributes().getRetAttrs(), true);
11322 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
11323
11324 SDNode *CallEnd = Result.second.getNode();
11325 if (CallEnd->getOpcode() == ISD::EH_LABEL)
11326 CallEnd = CallEnd->getOperand(0).getNode();
11327 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
11328 CallEnd = CallEnd->getOperand(0).getNode();
11329
11330 /// Get a call instruction from the call sequence chain.
11331 /// Tail calls are not allowed.
11332 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
11333 "Expected a callseq node.");
11334 SDNode *Call = CallEnd->getOperand(0).getNode();
11335 bool HasGlue = Call->getGluedNode();
11336
11337 // Replace the target specific call node with the patchable intrinsic.
11339
11340 // Push the chain.
11341 Ops.push_back(*(Call->op_begin()));
11342
11343 // Optionally, push the glue (if any).
11344 if (HasGlue)
11345 Ops.push_back(*(Call->op_end() - 1));
11346
11347 // Push the register mask info.
11348 if (HasGlue)
11349 Ops.push_back(*(Call->op_end() - 2));
11350 else
11351 Ops.push_back(*(Call->op_end() - 1));
11352
11353 // Add the <id> and <numBytes> constants.
11355 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
11357 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
11358
11359 // Add the callee.
11360 Ops.push_back(Callee);
11361
11362 // Adjust <numArgs> to account for any arguments that have been passed on the
11363 // stack instead.
11364 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
11365 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
11366 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11367 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
11368
11369 // Add the calling convention
11370 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
11371
11372 // Add the arguments we omitted previously. The register allocator should
11373 // place these in any free register.
11374 if (IsAnyRegCC)
11375 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
11376 Ops.push_back(getValue(CB.getArgOperand(i)));
11377
11378 // Push the arguments from the call instruction.
11379 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
11380 Ops.append(Call->op_begin() + 2, e);
11381
11382 // Push live variables for the stack map.
11383 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
11384
11385 SDVTList NodeTys;
11386 if (IsAnyRegCC && HasDef) {
11387 // Create the return types based on the intrinsic definition
11388 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11389 SmallVector<EVT, 3> ValueVTs;
11390 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
11391 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
11392
11393 // There is always a chain and a glue type at the end
11394 ValueVTs.push_back(MVT::Other);
11395 ValueVTs.push_back(MVT::Glue);
11396 NodeTys = DAG.getVTList(ValueVTs);
11397 } else
11398 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
11399
11400 // Replace the target specific call node with a PATCHPOINT node.
11401 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
11402
11403 // Update the NodeMap.
11404 if (HasDef) {
11405 if (IsAnyRegCC)
11406 setValue(&CB, SDValue(PPV.getNode(), 0));
11407 else
11408 setValue(&CB, Result.first);
11409 }
11410
11411 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
11412 // call sequence. Furthermore the location of the chain and glue can change
11413 // when the AnyReg calling convention is used and the intrinsic returns a
11414 // value.
11415 if (IsAnyRegCC && HasDef) {
11416 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
11417 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
11418 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
11419 } else
11420 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
11421 DAG.DeleteNode(Call);
11422
11423 // Inform the Frame Information that we have a patchpoint in this function.
11424 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11425}
11426
11427void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
11428 unsigned Intrinsic) {
11429 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11430 SDValue Op1 = getValue(I.getArgOperand(0));
11431 SDValue Op2;
11432 if (I.arg_size() > 1)
11433 Op2 = getValue(I.getArgOperand(1));
11434 SDLoc dl = getCurSDLoc();
11435 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
11436 SDValue Res;
11437 SDNodeFlags SDFlags;
11438 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
11439 SDFlags.copyFMF(*FPMO);
11440
11441 switch (Intrinsic) {
11442 case Intrinsic::vector_reduce_fadd:
11443 if (SDFlags.hasAllowReassociation())
11444 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
11445 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
11446 SDFlags);
11447 else
11448 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
11449 break;
11450 case Intrinsic::vector_reduce_fmul:
11451 if (SDFlags.hasAllowReassociation())
11452 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
11453 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
11454 SDFlags);
11455 else
11456 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
11457 break;
11458 case Intrinsic::vector_reduce_add:
11459 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
11460 break;
11461 case Intrinsic::vector_reduce_mul:
11462 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
11463 break;
11464 case Intrinsic::vector_reduce_and:
11465 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
11466 break;
11467 case Intrinsic::vector_reduce_or:
11468 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
11469 break;
11470 case Intrinsic::vector_reduce_xor:
11471 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
11472 break;
11473 case Intrinsic::vector_reduce_smax:
11474 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
11475 break;
11476 case Intrinsic::vector_reduce_smin:
11477 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
11478 break;
11479 case Intrinsic::vector_reduce_umax:
11480 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
11481 break;
11482 case Intrinsic::vector_reduce_umin:
11483 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
11484 break;
11485 case Intrinsic::vector_reduce_fmax:
11486 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
11487 break;
11488 case Intrinsic::vector_reduce_fmin:
11489 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
11490 break;
11491 case Intrinsic::vector_reduce_fmaximum:
11492 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
11493 break;
11494 case Intrinsic::vector_reduce_fminimum:
11495 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
11496 break;
11497 case Intrinsic::vector_reduce_fmaximumnum:
11498 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUMNUM, dl, VT, Op1, SDFlags);
11499 break;
11500 case Intrinsic::vector_reduce_fminimumnum:
11501 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUMNUM, dl, VT, Op1, SDFlags);
11502 break;
11503 default:
11504 llvm_unreachable("Unhandled vector reduce intrinsic");
11505 }
11506 setValue(&I, Res);
11507}
11508
11509/// Returns an AttributeList representing the attributes applied to the return
11510/// value of the given call.
11513 if (CLI.RetSExt)
11514 Attrs.push_back(Attribute::SExt);
11515 if (CLI.RetZExt)
11516 Attrs.push_back(Attribute::ZExt);
11517 if (CLI.IsInReg)
11518 Attrs.push_back(Attribute::InReg);
11519
11520 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
11521 Attrs);
11522}
11523
11524/// TargetLowering::LowerCallTo - This is the default LowerCallTo
11525/// implementation, which just calls LowerCall.
11526/// FIXME: When all targets are
11527/// migrated to using LowerCall, this hook should be integrated into SDISel.
11528std::pair<SDValue, SDValue>
11530 LLVMContext &Context = CLI.RetTy->getContext();
11531
11532 // Handle the incoming return values from the call.
11533 CLI.Ins.clear();
11534 SmallVector<Type *, 4> RetOrigTys;
11536 auto &DL = CLI.DAG.getDataLayout();
11537 ComputeValueTypes(DL, CLI.OrigRetTy, RetOrigTys, &Offsets);
11538
11539 SmallVector<EVT, 4> RetVTs;
11540 if (CLI.RetTy != CLI.OrigRetTy) {
11541 assert(RetOrigTys.size() == 1 &&
11542 "Only supported for non-aggregate returns");
11543 RetVTs.push_back(getValueType(DL, CLI.RetTy));
11544 } else {
11545 for (Type *Ty : RetOrigTys)
11546 RetVTs.push_back(getValueType(DL, Ty));
11547 }
11548
11549 if (CLI.IsPostTypeLegalization) {
11550 // If we are lowering a libcall after legalization, split the return type.
11551 SmallVector<Type *, 4> OldRetOrigTys;
11552 SmallVector<EVT, 4> OldRetVTs;
11553 SmallVector<TypeSize, 4> OldOffsets;
11554 RetOrigTys.swap(OldRetOrigTys);
11555 RetVTs.swap(OldRetVTs);
11556 Offsets.swap(OldOffsets);
11557
11558 for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
11559 EVT RetVT = OldRetVTs[i];
11560 uint64_t Offset = OldOffsets[i];
11561 MVT RegisterVT = getRegisterType(Context, RetVT);
11562 unsigned NumRegs = getNumRegisters(Context, RetVT);
11563 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
11564 RetOrigTys.append(NumRegs, OldRetOrigTys[i]);
11565 RetVTs.append(NumRegs, RegisterVT);
11566 for (unsigned j = 0; j != NumRegs; ++j)
11567 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
11568 }
11569 }
11570
11572 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
11573
11574 bool CanLowerReturn =
11576 CLI.IsVarArg, Outs, Context, CLI.RetTy);
11577
11578 SDValue DemoteStackSlot;
11579 int DemoteStackIdx = -100;
11580 if (!CanLowerReturn) {
11581 // FIXME: equivalent assert?
11582 // assert(!CS.hasInAllocaArgument() &&
11583 // "sret demotion is incompatible with inalloca");
11584 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
11585 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
11587 DemoteStackIdx =
11588 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
11589 Type *StackSlotPtrType = PointerType::get(Context, DL.getAllocaAddrSpace());
11590
11591 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
11592 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11593 Entry.IsSRet = true;
11594 Entry.Alignment = Alignment;
11595 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
11596 CLI.NumFixedArgs += 1;
11597 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11598 CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(Context);
11599
11600 // sret demotion isn't compatible with tail-calls, since the sret argument
11601 // points into the callers stack frame.
11602 CLI.IsTailCall = false;
11603 } else {
11604 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11605 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
11606 for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
11607 ISD::ArgFlagsTy Flags;
11608 if (NeedsRegBlock) {
11609 Flags.setInConsecutiveRegs();
11610 if (I == RetVTs.size() - 1)
11611 Flags.setInConsecutiveRegsLast();
11612 }
11613 EVT VT = RetVTs[I];
11614 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11615 unsigned NumRegs =
11616 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11617 for (unsigned i = 0; i != NumRegs; ++i) {
11618 ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
11620 if (CLI.RetTy->isPointerTy()) {
11621 Ret.Flags.setPointer();
11623 cast<PointerType>(CLI.RetTy)->getAddressSpace());
11624 }
11625 if (CLI.RetSExt)
11626 Ret.Flags.setSExt();
11627 if (CLI.RetZExt)
11628 Ret.Flags.setZExt();
11629 if (CLI.IsInReg)
11630 Ret.Flags.setInReg();
11631 CLI.Ins.push_back(Ret);
11632 }
11633 }
11634 }
11635
11636 // We push in swifterror return as the last element of CLI.Ins.
11637 ArgListTy &Args = CLI.getArgs();
11638 if (supportSwiftError()) {
11639 for (const ArgListEntry &Arg : Args) {
11640 if (Arg.IsSwiftError) {
11641 ISD::ArgFlagsTy Flags;
11642 Flags.setSwiftError();
11644 PointerType::getUnqual(Context),
11645 /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
11646 CLI.Ins.push_back(Ret);
11647 }
11648 }
11649 }
11650
11651 // Handle all of the outgoing arguments.
11652 CLI.Outs.clear();
11653 CLI.OutVals.clear();
11654 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11655 SmallVector<Type *, 4> OrigArgTys;
11656 ComputeValueTypes(DL, Args[i].OrigTy, OrigArgTys);
11657 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11658 Type *FinalType = Args[i].Ty;
11659 if (Args[i].IsByVal)
11660 FinalType = Args[i].IndirectType;
11661 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11662 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
11663 for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
11664 ++Value) {
11665 Type *OrigArgTy = OrigArgTys[Value];
11666 Type *ArgTy = OrigArgTy;
11667 if (Args[i].Ty != Args[i].OrigTy) {
11668 assert(Value == 0 && "Only supported for non-aggregate arguments");
11669 ArgTy = Args[i].Ty;
11670 }
11671
11672 EVT VT = getValueType(DL, ArgTy);
11673 SDValue Op = SDValue(Args[i].Node.getNode(),
11674 Args[i].Node.getResNo() + Value);
11675 ISD::ArgFlagsTy Flags;
11676
11677 // Certain targets (such as MIPS), may have a different ABI alignment
11678 // for a type depending on the context. Give the target a chance to
11679 // specify the alignment it wants.
11680 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11681 Flags.setOrigAlign(OriginalAlignment);
11682
11683 if (i >= CLI.NumFixedArgs)
11684 Flags.setVarArg();
11685 if (ArgTy->isPointerTy()) {
11686 Flags.setPointer();
11687 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
11688 }
11689 if (Args[i].IsZExt)
11690 Flags.setZExt();
11691 if (Args[i].IsSExt)
11692 Flags.setSExt();
11693 if (Args[i].IsNoExt)
11694 Flags.setNoExt();
11695 if (Args[i].IsInReg) {
11696 // If we are using vectorcall calling convention, a structure that is
11697 // passed InReg - is surely an HVA
11699 isa<StructType>(FinalType)) {
11700 // The first value of a structure is marked
11701 if (0 == Value)
11702 Flags.setHvaStart();
11703 Flags.setHva();
11704 }
11705 // Set InReg Flag
11706 Flags.setInReg();
11707 }
11708 if (Args[i].IsSRet)
11709 Flags.setSRet();
11710 if (Args[i].IsSwiftSelf)
11711 Flags.setSwiftSelf();
11712 if (Args[i].IsSwiftAsync)
11713 Flags.setSwiftAsync();
11714 if (Args[i].IsSwiftError)
11715 Flags.setSwiftError();
11716 if (Args[i].IsCFGuardTarget)
11717 Flags.setCFGuardTarget();
11718 if (Args[i].IsByVal)
11719 Flags.setByVal();
11720 if (Args[i].IsByRef)
11721 Flags.setByRef();
11722 if (Args[i].IsPreallocated) {
11723 Flags.setPreallocated();
11724 // Set the byval flag for CCAssignFn callbacks that don't know about
11725 // preallocated. This way we can know how many bytes we should've
11726 // allocated and how many bytes a callee cleanup function will pop. If
11727 // we port preallocated to more targets, we'll have to add custom
11728 // preallocated handling in the various CC lowering callbacks.
11729 Flags.setByVal();
11730 }
11731 if (Args[i].IsInAlloca) {
11732 Flags.setInAlloca();
11733 // Set the byval flag for CCAssignFn callbacks that don't know about
11734 // inalloca. This way we can know how many bytes we should've allocated
11735 // and how many bytes a callee cleanup function will pop. If we port
11736 // inalloca to more targets, we'll have to add custom inalloca handling
11737 // in the various CC lowering callbacks.
11738 Flags.setByVal();
11739 }
11740 Align MemAlign;
11741 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11742 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
11743 Flags.setByValSize(FrameSize);
11744
11745 // info is not there but there are cases it cannot get right.
11746 if (auto MA = Args[i].Alignment)
11747 MemAlign = *MA;
11748 else
11749 MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL);
11750 } else if (auto MA = Args[i].Alignment) {
11751 MemAlign = *MA;
11752 } else {
11753 MemAlign = OriginalAlignment;
11754 }
11755 Flags.setMemAlign(MemAlign);
11756 if (Args[i].IsNest)
11757 Flags.setNest();
11758 if (NeedsRegBlock)
11759 Flags.setInConsecutiveRegs();
11760
11761 MVT PartVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11762 unsigned NumParts =
11763 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11766
11767 if (Args[i].IsSExt)
11769 else if (Args[i].IsZExt)
11771
11772 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11773 // for now.
11774 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11776 assert((CLI.RetTy == Args[i].Ty ||
11777 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11779 Args[i].Ty->getPointerAddressSpace())) &&
11780 RetVTs.size() == NumValues && "unexpected use of 'returned'");
11781 // Before passing 'returned' to the target lowering code, ensure that
11782 // either the register MVT and the actual EVT are the same size or that
11783 // the return value and argument are extended in the same way; in these
11784 // cases it's safe to pass the argument register value unchanged as the
11785 // return register value (although it's at the target's option whether
11786 // to do so)
11787 // TODO: allow code generation to take advantage of partially preserved
11788 // registers rather than clobbering the entire register when the
11789 // parameter extension method is not compatible with the return
11790 // extension method
11791 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11792 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11793 CLI.RetZExt == Args[i].IsZExt))
11794 Flags.setReturned();
11795 }
11796
11797 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
11798 CLI.CallConv, ExtendKind);
11799
11800 for (unsigned j = 0; j != NumParts; ++j) {
11801 // if it isn't first piece, alignment must be 1
11802 // For scalable vectors the scalable part is currently handled
11803 // by individual targets, so we just use the known minimum size here.
11804 ISD::OutputArg MyFlags(
11805 Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
11806 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11807 if (NumParts > 1 && j == 0)
11808 MyFlags.Flags.setSplit();
11809 else if (j != 0) {
11810 MyFlags.Flags.setOrigAlign(Align(1));
11811 if (j == NumParts - 1)
11812 MyFlags.Flags.setSplitEnd();
11813 }
11814
11815 CLI.Outs.push_back(MyFlags);
11816 CLI.OutVals.push_back(Parts[j]);
11817 }
11818
11819 if (NeedsRegBlock && Value == NumValues - 1)
11820 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11821 }
11822 }
11823
11825 CLI.Chain = LowerCall(CLI, InVals);
11826
11827 // Update CLI.InVals to use outside of this function.
11828 CLI.InVals = InVals;
11829
11830 // Verify that the target's LowerCall behaved as expected.
11831 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11832 "LowerCall didn't return a valid chain!");
11833 assert((!CLI.IsTailCall || InVals.empty()) &&
11834 "LowerCall emitted a return value for a tail call!");
11835 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11836 "LowerCall didn't emit the correct number of values!");
11837
11838 // For a tail call, the return value is merely live-out and there aren't
11839 // any nodes in the DAG representing it. Return a special value to
11840 // indicate that a tail call has been emitted and no more Instructions
11841 // should be processed in the current block.
11842 if (CLI.IsTailCall) {
11843 CLI.DAG.setRoot(CLI.Chain);
11844 return std::make_pair(SDValue(), SDValue());
11845 }
11846
11847#ifndef NDEBUG
11848 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11849 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11850 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11851 "LowerCall emitted a value with the wrong type!");
11852 }
11853#endif
11854
11855 SmallVector<SDValue, 4> ReturnValues;
11856 if (!CanLowerReturn) {
11857 // The instruction result is the result of loading from the
11858 // hidden sret parameter.
11859 MVT PtrVT = getPointerTy(DL, DL.getAllocaAddrSpace());
11860
11861 unsigned NumValues = RetVTs.size();
11862 ReturnValues.resize(NumValues);
11863 SmallVector<SDValue, 4> Chains(NumValues);
11864
11865 // An aggregate return value cannot wrap around the address space, so
11866 // offsets to its parts don't wrap either.
11868 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
11869 for (unsigned i = 0; i < NumValues; ++i) {
11871 DemoteStackSlot, CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT),
11873 SDValue L = CLI.DAG.getLoad(
11874 RetVTs[i], CLI.DL, CLI.Chain, Add,
11876 DemoteStackIdx, Offsets[i]),
11877 HiddenSRetAlign);
11878 ReturnValues[i] = L;
11879 Chains[i] = L.getValue(1);
11880 }
11881
11882 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
11883 } else {
11884 // Collect the legal value parts into potentially illegal values
11885 // that correspond to the original function's return values.
11886 std::optional<ISD::NodeType> AssertOp;
11887 if (CLI.RetSExt)
11888 AssertOp = ISD::AssertSext;
11889 else if (CLI.RetZExt)
11890 AssertOp = ISD::AssertZext;
11891 unsigned CurReg = 0;
11892 for (EVT VT : RetVTs) {
11893 MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
11894 unsigned NumRegs =
11895 getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
11896
11897 ReturnValues.push_back(getCopyFromParts(
11898 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
11899 CLI.Chain, CLI.CallConv, AssertOp));
11900 CurReg += NumRegs;
11901 }
11902
11903 // For a function returning void, there is no return value. We can't create
11904 // such a node, so we just return a null return value in that case. In
11905 // that case, nothing will actually look at the value.
11906 if (ReturnValues.empty())
11907 return std::make_pair(SDValue(), CLI.Chain);
11908 }
11909
11910 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
11911 CLI.DAG.getVTList(RetVTs), ReturnValues);
11912 return std::make_pair(Res, CLI.Chain);
11913}
11914
11915/// Places new result values for the node in Results (their number
11916/// and types must exactly match those of the original return values of
11917/// the node), or leaves Results empty, which indicates that the node is not
11918/// to be custom lowered after all.
11921 SelectionDAG &DAG) const {
11922 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
11923
11924 if (!Res.getNode())
11925 return;
11926
11927 // If the original node has one result, take the return value from
11928 // LowerOperation as is. It might not be result number 0.
11929 if (N->getNumValues() == 1) {
11930 Results.push_back(Res);
11931 return;
11932 }
11933
11934 // If the original node has multiple results, then the return node should
11935 // have the same number of results.
11936 assert((N->getNumValues() == Res->getNumValues()) &&
11937 "Lowering returned the wrong number of results!");
11938
11939 // Places new result values base on N result number.
11940 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11941 Results.push_back(Res.getValue(I));
11942}
11943
11945 llvm_unreachable("LowerOperation not implemented for this target!");
11946}
11947
11949 Register Reg,
11950 ISD::NodeType ExtendType) {
11952 assert((Op.getOpcode() != ISD::CopyFromReg ||
11953 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11954 "Copy from a reg to the same reg!");
11955 assert(!Reg.isPhysical() && "Is a physreg");
11956
11957 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11958 // If this is an InlineAsm we have to match the registers required, not the
11959 // notional registers required by the type.
11960
11961 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11962 std::nullopt); // This is not an ABI copy.
11963 SDValue Chain = DAG.getEntryNode();
11964
11965 if (ExtendType == ISD::ANY_EXTEND) {
11966 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
11967 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11968 ExtendType = PreferredExtendIt->second;
11969 }
11970 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
11971 PendingExports.push_back(Chain);
11972}
11973
11975
11976/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11977/// entry block, return true. This includes arguments used by switches, since
11978/// the switch may expand into multiple basic blocks.
11979static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11980 // With FastISel active, we may be splitting blocks, so force creation
11981 // of virtual registers for all non-dead arguments.
11982 if (FastISel)
11983 return A->use_empty();
11984
11985 const BasicBlock &Entry = A->getParent()->front();
11986 for (const User *U : A->users())
11987 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
11988 return false; // Use not in entry block.
11989
11990 return true;
11991}
11992
11994 DenseMap<const Argument *,
11995 std::pair<const AllocaInst *, const StoreInst *>>;
11996
11997/// Scan the entry block of the function in FuncInfo for arguments that look
11998/// like copies into a local alloca. Record any copied arguments in
11999/// ArgCopyElisionCandidates.
12000static void
12002 FunctionLoweringInfo *FuncInfo,
12003 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
12004 // Record the state of every static alloca used in the entry block. Argument
12005 // allocas are all used in the entry block, so we need approximately as many
12006 // entries as we have arguments.
12007 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
12009 unsigned NumArgs = FuncInfo->Fn->arg_size();
12010 StaticAllocas.reserve(NumArgs * 2);
12011
12012 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
12013 if (!V)
12014 return nullptr;
12015 V = V->stripPointerCasts();
12016 const auto *AI = dyn_cast<AllocaInst>(V);
12017 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
12018 return nullptr;
12019 auto Iter = StaticAllocas.insert({AI, Unknown});
12020 return &Iter.first->second;
12021 };
12022
12023 // Look for stores of arguments to static allocas. Look through bitcasts and
12024 // GEPs to handle type coercions, as long as the alloca is fully initialized
12025 // by the store. Any non-store use of an alloca escapes it and any subsequent
12026 // unanalyzed store might write it.
12027 // FIXME: Handle structs initialized with multiple stores.
12028 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
12029 // Look for stores, and handle non-store uses conservatively.
12030 const auto *SI = dyn_cast<StoreInst>(&I);
12031 if (!SI) {
12032 // We will look through cast uses, so ignore them completely.
12033 if (I.isCast())
12034 continue;
12035 // Ignore pseudo op intrinsics, they don't escape or store to allocas.
12037 continue;
12038 // This is an unknown instruction. Assume it escapes or writes to all
12039 // static alloca operands.
12040 for (const Use &U : I.operands()) {
12041 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
12042 *Info = StaticAllocaInfo::Clobbered;
12043 }
12044 continue;
12045 }
12046
12047 // If the stored value is a static alloca, mark it as escaped.
12048 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
12049 *Info = StaticAllocaInfo::Clobbered;
12050
12051 // Check if the destination is a static alloca.
12052 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
12053 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
12054 if (!Info)
12055 continue;
12056 const AllocaInst *AI = cast<AllocaInst>(Dst);
12057
12058 // Skip allocas that have been initialized or clobbered.
12059 if (*Info != StaticAllocaInfo::Unknown)
12060 continue;
12061
12062 // Check if the stored value is an argument, and that this store fully
12063 // initializes the alloca.
12064 // If the argument type has padding bits we can't directly forward a pointer
12065 // as the upper bits may contain garbage.
12066 // Don't elide copies from the same argument twice.
12067 const Value *Val = SI->getValueOperand()->stripPointerCasts();
12068 const auto *Arg = dyn_cast<Argument>(Val);
12069 std::optional<TypeSize> AllocaSize = AI->getAllocationSize(DL);
12070 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
12071 Arg->getType()->isEmptyTy() || !AllocaSize ||
12072 DL.getTypeStoreSize(Arg->getType()) != *AllocaSize ||
12073 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
12074 ArgCopyElisionCandidates.count(Arg)) {
12075 *Info = StaticAllocaInfo::Clobbered;
12076 continue;
12077 }
12078
12079 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
12080 << '\n');
12081
12082 // Mark this alloca and store for argument copy elision.
12083 *Info = StaticAllocaInfo::Elidable;
12084 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
12085
12086 // Stop scanning if we've seen all arguments. This will happen early in -O0
12087 // builds, which is useful, because -O0 builds have large entry blocks and
12088 // many allocas.
12089 if (ArgCopyElisionCandidates.size() == NumArgs)
12090 break;
12091 }
12092}
12093
12094/// Try to elide argument copies from memory into a local alloca. Succeeds if
12095/// ArgVal is a load from a suitable fixed stack object.
12098 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
12099 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
12100 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
12101 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
12102 // Check if this is a load from a fixed stack object.
12103 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
12104 if (!LNode)
12105 return;
12106 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
12107 if (!FINode)
12108 return;
12109
12110 // Check that the fixed stack object is the right size and alignment.
12111 // Look at the alignment that the user wrote on the alloca instead of looking
12112 // at the stack object.
12113 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
12114 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
12115 const AllocaInst *AI = ArgCopyIter->second.first;
12116 int FixedIndex = FINode->getIndex();
12117 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
12118 int OldIndex = AllocaIndex;
12119 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
12120 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
12121 LLVM_DEBUG(
12122 dbgs() << " argument copy elision failed due to bad fixed stack "
12123 "object size\n");
12124 return;
12125 }
12126 Align RequiredAlignment = AI->getAlign();
12127 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
12128 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
12129 "greater than stack argument alignment ("
12130 << DebugStr(RequiredAlignment) << " vs "
12131 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
12132 return;
12133 }
12134
12135 // Perform the elision. Delete the old stack object and replace its only use
12136 // in the variable info map. Mark the stack object as mutable and aliased.
12137 LLVM_DEBUG({
12138 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
12139 << " Replacing frame index " << OldIndex << " with " << FixedIndex
12140 << '\n';
12141 });
12142 MFI.RemoveStackObject(OldIndex);
12143 MFI.setIsImmutableObjectIndex(FixedIndex, false);
12144 MFI.setIsAliasedObjectIndex(FixedIndex, true);
12145 AllocaIndex = FixedIndex;
12146 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
12147 for (SDValue ArgVal : ArgVals)
12148 Chains.push_back(ArgVal.getValue(1));
12149
12150 // Avoid emitting code for the store implementing the copy.
12151 const StoreInst *SI = ArgCopyIter->second.second;
12152 ElidedArgCopyInstrs.insert(SI);
12153
12154 // Check for uses of the argument again so that we can avoid exporting ArgVal
12155 // if it is't used by anything other than the store.
12156 for (const Value *U : Arg.users()) {
12157 if (U != SI) {
12158 ArgHasUses = true;
12159 break;
12160 }
12161 }
12162}
12163
12164void SelectionDAGISel::LowerArguments(const Function &F, bool FastISel) {
12165 SelectionDAG &DAG = SDB->DAG;
12166 SDLoc dl = SDB->getCurSDLoc();
12167 const DataLayout &DL = DAG.getDataLayout();
12169
12170 // In Naked functions we aren't going to save any registers.
12171 if (F.hasFnAttribute(Attribute::Naked))
12172 return;
12173
12174 if (!FuncInfo->CanLowerReturn) {
12175 // Put in an sret pointer parameter before all the other parameters.
12176 MVT ValueVT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
12177
12178 ISD::ArgFlagsTy Flags;
12179 Flags.setSRet();
12180 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVT);
12181 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
12183 Ins.push_back(RetArg);
12184 }
12185
12186 // Look for stores of arguments to static allocas. Mark such arguments with a
12187 // flag to ask the target to give us the memory location of that argument if
12188 // available.
12189 ArgCopyElisionMapTy ArgCopyElisionCandidates;
12191 ArgCopyElisionCandidates);
12192
12193 // Set up the incoming argument description vector.
12194 for (const Argument &Arg : F.args()) {
12195 unsigned ArgNo = Arg.getArgNo();
12197 ComputeValueTypes(DAG.getDataLayout(), Arg.getType(), Types);
12198 bool isArgValueUsed = !Arg.use_empty();
12199 Type *FinalType = Arg.getType();
12200 if (Arg.hasAttribute(Attribute::ByVal))
12201 FinalType = Arg.getParamByValType();
12202 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
12203 FinalType, F.getCallingConv(), F.isVarArg(), DL);
12204 for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
12205 ++Value) {
12206 Type *ArgTy = Types[Value];
12207 EVT VT = TLI->getValueType(DL, ArgTy);
12208 ISD::ArgFlagsTy Flags;
12209
12210 if (ArgTy->isPointerTy()) {
12211 Flags.setPointer();
12212 Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
12213 }
12214 if (Arg.hasAttribute(Attribute::ZExt))
12215 Flags.setZExt();
12216 if (Arg.hasAttribute(Attribute::SExt))
12217 Flags.setSExt();
12218 if (Arg.hasAttribute(Attribute::InReg)) {
12219 // If we are using vectorcall calling convention, a structure that is
12220 // passed InReg - is surely an HVA
12221 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
12222 isa<StructType>(Arg.getType())) {
12223 // The first value of a structure is marked
12224 if (0 == Value)
12225 Flags.setHvaStart();
12226 Flags.setHva();
12227 }
12228 // Set InReg Flag
12229 Flags.setInReg();
12230 }
12231 if (Arg.hasAttribute(Attribute::StructRet))
12232 Flags.setSRet();
12233 if (Arg.hasAttribute(Attribute::SwiftSelf))
12234 Flags.setSwiftSelf();
12235 if (Arg.hasAttribute(Attribute::SwiftAsync))
12236 Flags.setSwiftAsync();
12237 if (Arg.hasAttribute(Attribute::SwiftError))
12238 Flags.setSwiftError();
12239 if (Arg.hasAttribute(Attribute::ByVal))
12240 Flags.setByVal();
12241 if (Arg.hasAttribute(Attribute::ByRef))
12242 Flags.setByRef();
12243 if (Arg.hasAttribute(Attribute::InAlloca)) {
12244 Flags.setInAlloca();
12245 // Set the byval flag for CCAssignFn callbacks that don't know about
12246 // inalloca. This way we can know how many bytes we should've allocated
12247 // and how many bytes a callee cleanup function will pop. If we port
12248 // inalloca to more targets, we'll have to add custom inalloca handling
12249 // in the various CC lowering callbacks.
12250 Flags.setByVal();
12251 }
12252 if (Arg.hasAttribute(Attribute::Preallocated)) {
12253 Flags.setPreallocated();
12254 // Set the byval flag for CCAssignFn callbacks that don't know about
12255 // preallocated. This way we can know how many bytes we should've
12256 // allocated and how many bytes a callee cleanup function will pop. If
12257 // we port preallocated to more targets, we'll have to add custom
12258 // preallocated handling in the various CC lowering callbacks.
12259 Flags.setByVal();
12260 }
12261
12262 // Certain targets (such as MIPS), may have a different ABI alignment
12263 // for a type depending on the context. Give the target a chance to
12264 // specify the alignment it wants.
12265 const Align OriginalAlignment(
12266 TLI->getABIAlignmentForCallingConv(ArgTy, DL));
12267 Flags.setOrigAlign(OriginalAlignment);
12268
12269 Align MemAlign;
12270 Type *ArgMemTy = nullptr;
12271 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
12272 Flags.isByRef()) {
12273 if (!ArgMemTy)
12274 ArgMemTy = Arg.getPointeeInMemoryValueType();
12275
12276 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
12277
12278 // For in-memory arguments, size and alignment should be passed from FE.
12279 // BE will guess if this info is not there but there are cases it cannot
12280 // get right.
12281 if (auto ParamAlign = Arg.getParamStackAlign())
12282 MemAlign = *ParamAlign;
12283 else if ((ParamAlign = Arg.getParamAlign()))
12284 MemAlign = *ParamAlign;
12285 else
12286 MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL);
12287 if (Flags.isByRef())
12288 Flags.setByRefSize(MemSize);
12289 else
12290 Flags.setByValSize(MemSize);
12291 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
12292 MemAlign = *ParamAlign;
12293 } else {
12294 MemAlign = OriginalAlignment;
12295 }
12296 Flags.setMemAlign(MemAlign);
12297
12298 if (Arg.hasAttribute(Attribute::Nest))
12299 Flags.setNest();
12300 if (NeedsRegBlock)
12301 Flags.setInConsecutiveRegs();
12302 if (ArgCopyElisionCandidates.count(&Arg))
12303 Flags.setCopyElisionCandidate();
12304 if (Arg.hasAttribute(Attribute::Returned))
12305 Flags.setReturned();
12306
12307 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
12308 *CurDAG->getContext(), F.getCallingConv(), VT);
12309 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
12310 *CurDAG->getContext(), F.getCallingConv(), VT);
12311 for (unsigned i = 0; i != NumRegs; ++i) {
12312 // For scalable vectors, use the minimum size; individual targets
12313 // are responsible for handling scalable vector arguments and
12314 // return values.
12315 ISD::InputArg MyFlags(
12316 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12317 i * RegisterVT.getStoreSize().getKnownMinValue());
12318 if (NumRegs > 1 && i == 0)
12319 MyFlags.Flags.setSplit();
12320 // if it isn't first piece, alignment must be 1
12321 else if (i > 0) {
12322 MyFlags.Flags.setOrigAlign(Align(1));
12323 if (i == NumRegs - 1)
12324 MyFlags.Flags.setSplitEnd();
12325 }
12326 Ins.push_back(MyFlags);
12327 }
12328 if (NeedsRegBlock && Value == NumValues - 1)
12329 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
12330 }
12331 }
12332
12333 // Call the target to set up the argument values.
12335 SDValue NewRoot = TLI->LowerFormalArguments(
12336 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
12337
12338 // Verify that the target's LowerFormalArguments behaved as expected.
12339 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
12340 "LowerFormalArguments didn't return a valid chain!");
12341 assert(InVals.size() == Ins.size() &&
12342 "LowerFormalArguments didn't emit the correct number of values!");
12343 assert(all_of(InVals, [](SDValue InVal) { return InVal.getNode(); }) &&
12344 "LowerFormalArguments emitted a null value!");
12345
12346 // Update the DAG with the new chain value resulting from argument lowering.
12347 DAG.setRoot(NewRoot);
12348
12349 // Set up the argument values.
12350 unsigned i = 0;
12351 if (!FuncInfo->CanLowerReturn) {
12352 // Create a virtual register for the sret pointer, and put in a copy
12353 // from the sret argument into it.
12354 MVT VT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
12355 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
12356 std::optional<ISD::NodeType> AssertOp;
12357 SDValue ArgValue =
12358 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
12359 F.getCallingConv(), AssertOp);
12360
12361 MachineFunction& MF = SDB->DAG.getMachineFunction();
12362 MachineRegisterInfo& RegInfo = MF.getRegInfo();
12363 Register SRetReg =
12364 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
12365 FuncInfo->DemoteRegister = SRetReg;
12366 NewRoot =
12367 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
12368 DAG.setRoot(NewRoot);
12369
12370 // i indexes lowered arguments. Bump it past the hidden sret argument.
12371 ++i;
12372 }
12373
12375 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12376 for (const Argument &Arg : F.args()) {
12377 SmallVector<SDValue, 4> ArgValues;
12378 SmallVector<EVT, 4> ValueVTs;
12379 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
12380 unsigned NumValues = ValueVTs.size();
12381 if (NumValues == 0)
12382 continue;
12383
12384 bool ArgHasUses = !Arg.use_empty();
12385
12386 // Elide the copying store if the target loaded this argument from a
12387 // suitable fixed stack object.
12388 if (Ins[i].Flags.isCopyElisionCandidate()) {
12389 unsigned NumParts = 0;
12390 for (EVT VT : ValueVTs)
12391 NumParts += TLI->getNumRegistersForCallingConv(*CurDAG->getContext(),
12392 F.getCallingConv(), VT);
12393
12394 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
12395 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
12396 ArrayRef(&InVals[i], NumParts), ArgHasUses);
12397 }
12398
12399 // If this argument is unused then remember its value. It is used to generate
12400 // debugging information.
12401 bool isSwiftErrorArg =
12402 TLI->supportSwiftError() &&
12403 Arg.hasAttribute(Attribute::SwiftError);
12404 if (!ArgHasUses && !isSwiftErrorArg) {
12405 SDB->setUnusedArgValue(&Arg, InVals[i]);
12406
12407 // Also remember any frame index for use in FastISel.
12408 if (FrameIndexSDNode *FI =
12410 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12411 }
12412
12413 for (unsigned Val = 0; Val != NumValues; ++Val) {
12414 EVT VT = ValueVTs[Val];
12415 MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
12416 F.getCallingConv(), VT);
12417 unsigned NumParts = TLI->getNumRegistersForCallingConv(
12418 *CurDAG->getContext(), F.getCallingConv(), VT);
12419
12420 // Even an apparent 'unused' swifterror argument needs to be returned. So
12421 // we do generate a copy for it that can be used on return from the
12422 // function.
12423 if (ArgHasUses || isSwiftErrorArg) {
12424 std::optional<ISD::NodeType> AssertOp;
12425 if (Arg.hasAttribute(Attribute::SExt))
12426 AssertOp = ISD::AssertSext;
12427 else if (Arg.hasAttribute(Attribute::ZExt))
12428 AssertOp = ISD::AssertZext;
12429
12430 SDValue OutVal =
12431 getCopyFromParts(DAG, dl, &InVals[i], NumParts, PartVT, VT, nullptr,
12432 NewRoot, F.getCallingConv(), AssertOp);
12433
12434 FPClassTest NoFPClass = Arg.getNoFPClass();
12435 if (NoFPClass != fcNone) {
12436 SDValue SDNoFPClass = DAG.getTargetConstant(
12437 static_cast<uint64_t>(NoFPClass), dl, MVT::i32);
12438 OutVal = DAG.getNode(ISD::AssertNoFPClass, dl, OutVal.getValueType(),
12439 OutVal, SDNoFPClass);
12440 }
12441 ArgValues.push_back(OutVal);
12442 }
12443
12444 i += NumParts;
12445 }
12446
12447 // We don't need to do anything else for unused arguments.
12448 if (ArgValues.empty())
12449 continue;
12450
12451 // Note down frame index.
12452 if (FrameIndexSDNode *FI =
12453 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
12454 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12455
12456 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
12457 SDB->getCurSDLoc());
12458
12459 SDB->setValue(&Arg, Res);
12460 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
12461 // We want to associate the argument with the frame index, among
12462 // involved operands, that correspond to the lowest address. The
12463 // getCopyFromParts function, called earlier, is swapping the order of
12464 // the operands to BUILD_PAIR depending on endianness. The result of
12465 // that swapping is that the least significant bits of the argument will
12466 // be in the first operand of the BUILD_PAIR node, and the most
12467 // significant bits will be in the second operand.
12468 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
12469 if (LoadSDNode *LNode =
12470 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
12471 if (FrameIndexSDNode *FI =
12472 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
12473 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12474 }
12475
12476 // Analyses past this point are naive and don't expect an assertion.
12477 if (Res.getOpcode() == ISD::AssertZext)
12478 Res = Res.getOperand(0);
12479
12480 // Update the SwiftErrorVRegDefMap.
12481 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
12482 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
12483 if (Reg.isVirtual())
12484 SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
12485 Reg);
12486 }
12487
12488 // If this argument is live outside of the entry block, insert a copy from
12489 // wherever we got it to the vreg that other BB's will reference it as.
12490 if (Res.getOpcode() == ISD::CopyFromReg) {
12491 // If we can, though, try to skip creating an unnecessary vreg.
12492 // FIXME: This isn't very clean... it would be nice to make this more
12493 // general.
12494 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
12495 if (Reg.isVirtual()) {
12496 FuncInfo->ValueMap[&Arg] = Reg;
12497 continue;
12498 }
12499 }
12500 if (!isOnlyUsedInEntryBlock(&Arg, FastISel)) {
12501 FuncInfo->InitializeRegForValue(&Arg);
12502 SDB->CopyToExportRegsIfNeeded(&Arg);
12503 }
12504 }
12505
12506 if (!Chains.empty()) {
12507 Chains.push_back(NewRoot);
12508 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
12509 }
12510
12511 DAG.setRoot(NewRoot);
12512
12513 assert(i == InVals.size() && "Argument register count mismatch!");
12514
12515 // If any argument copy elisions occurred and we have debug info, update the
12516 // stale frame indices used in the dbg.declare variable info table.
12517 if (!ArgCopyElisionFrameIndexMap.empty()) {
12518 for (MachineFunction::VariableDbgInfo &VI :
12519 MF->getInStackSlotVariableDbgInfo()) {
12520 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
12521 if (I != ArgCopyElisionFrameIndexMap.end())
12522 VI.updateStackSlot(I->second);
12523 }
12524 }
12525
12526 // Finally, if the target has anything special to do, allow it to do so.
12528}
12529
12530/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
12531/// ensure constants are generated when needed. Remember the virtual registers
12532/// that need to be added to the Machine PHI nodes as input. We cannot just
12533/// directly add them, because expansion might result in multiple MBB's for one
12534/// BB. As such, the start of the BB might correspond to a different MBB than
12535/// the end.
12536void
12537SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
12538 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12539
12540 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12541
12542 // Check PHI nodes in successors that expect a value to be available from this
12543 // block.
12544 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
12545 if (!isa<PHINode>(SuccBB->begin())) continue;
12546 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB);
12547
12548 // If this terminator has multiple identical successors (common for
12549 // switches), only handle each succ once.
12550 if (!SuccsHandled.insert(SuccMBB).second)
12551 continue;
12552
12554
12555 // At this point we know that there is a 1-1 correspondence between LLVM PHI
12556 // nodes and Machine PHI nodes, but the incoming operands have not been
12557 // emitted yet.
12558 for (const PHINode &PN : SuccBB->phis()) {
12559 // Ignore dead phi's.
12560 if (PN.use_empty())
12561 continue;
12562
12563 // Skip empty types
12564 if (PN.getType()->isEmptyTy())
12565 continue;
12566
12567 Register Reg;
12568 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
12569
12570 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
12571 Register &RegOut = ConstantsOut[C];
12572 if (!RegOut) {
12573 RegOut = FuncInfo.CreateRegs(PHIOp);
12574 // We need to zero/sign extend ConstantInt phi operands to match
12575 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12576 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12577 if (auto *CI = dyn_cast<ConstantInt>(C))
12578 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
12580 CopyValueToVirtualRegister(C, RegOut, ExtendType);
12581 }
12582 Reg = RegOut;
12583 } else {
12584 auto I = FuncInfo.ValueMap.find(PHIOp);
12585 if (I != FuncInfo.ValueMap.end())
12586 Reg = I->second;
12587 else {
12588 assert(isa<AllocaInst>(PHIOp) &&
12589 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12590 "Didn't codegen value into a register!??");
12591 Reg = FuncInfo.CreateRegs(PHIOp);
12593 }
12594 }
12595
12596 // Remember that this register needs to added to the machine PHI node as
12597 // the input for this MBB.
12598 SmallVector<EVT, 4> ValueVTs;
12599 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
12600 for (EVT VT : ValueVTs) {
12601 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
12602 for (unsigned i = 0; i != NumRegisters; ++i)
12603 FuncInfo.PHINodesToUpdate.emplace_back(&*MBBI++, Reg + i);
12604 Reg += NumRegisters;
12605 }
12606 }
12607 }
12608
12609 ConstantsOut.clear();
12610}
12611
12612MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12614 if (++I == FuncInfo.MF->end())
12615 return nullptr;
12616 return &*I;
12617}
12618
12619/// During lowering new call nodes can be created (such as memset, etc.).
12620/// Those will become new roots of the current DAG, but complications arise
12621/// when they are tail calls. In such cases, the call lowering will update
12622/// the root, but the builder still needs to know that a tail call has been
12623/// lowered in order to avoid generating an additional return.
12624void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12625 // If the node is null, we do have a tail call.
12626 if (MaybeTC.getNode() != nullptr)
12627 DAG.setRoot(MaybeTC);
12628 else
12629 HasTailCall = true;
12630}
12631
12632void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12633 MachineBasicBlock *SwitchMBB,
12634 MachineBasicBlock *DefaultMBB) {
12635 MachineFunction *CurMF = FuncInfo.MF;
12636 MachineBasicBlock *NextMBB = nullptr;
12638 if (++BBI != FuncInfo.MF->end())
12639 NextMBB = &*BBI;
12640
12641 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12642
12643 BranchProbabilityInfo *BPI = FuncInfo.BPI;
12644
12645 if (Size == 2 && W.MBB == SwitchMBB) {
12646 // If any two of the cases has the same destination, and if one value
12647 // is the same as the other, but has one bit unset that the other has set,
12648 // use bit manipulation to do two compares at once. For example:
12649 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12650 // TODO: This could be extended to merge any 2 cases in switches with 3
12651 // cases.
12652 // TODO: Handle cases where W.CaseBB != SwitchBB.
12653 CaseCluster &Small = *W.FirstCluster;
12654 CaseCluster &Big = *W.LastCluster;
12655
12656 if (Small.Low == Small.High && Big.Low == Big.High &&
12657 Small.MBB == Big.MBB) {
12658 const APInt &SmallValue = Small.Low->getValue();
12659 const APInt &BigValue = Big.Low->getValue();
12660
12661 // Check that there is only one bit different.
12662 APInt CommonBit = BigValue ^ SmallValue;
12663 if (CommonBit.isPowerOf2()) {
12664 SDValue CondLHS = getValue(Cond);
12665 EVT VT = CondLHS.getValueType();
12666 SDLoc DL = getCurSDLoc();
12667
12668 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
12669 DAG.getConstant(CommonBit, DL, VT));
12670 SDValue Cond = DAG.getSetCC(
12671 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
12672 ISD::SETEQ);
12673
12674 // Update successor info.
12675 // Both Small and Big will jump to Small.BB, so we sum up the
12676 // probabilities.
12677 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
12678 if (BPI)
12679 addSuccessorWithProb(
12680 SwitchMBB, DefaultMBB,
12681 // The default destination is the first successor in IR.
12682 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
12683 else
12684 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12685
12686 // Insert the true branch.
12687 SDValue BrCond =
12688 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
12689 DAG.getBasicBlock(Small.MBB));
12690 // Insert the false branch.
12691 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
12692 DAG.getBasicBlock(DefaultMBB));
12693
12694 DAG.setRoot(BrCond);
12695 return;
12696 }
12697 }
12698 }
12699
12700 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12701 // Here, we order cases by probability so the most likely case will be
12702 // checked first. However, two clusters can have the same probability in
12703 // which case their relative ordering is non-deterministic. So we use Low
12704 // as a tie-breaker as clusters are guaranteed to never overlap.
12705 llvm::sort(W.FirstCluster, W.LastCluster + 1,
12706 [](const CaseCluster &a, const CaseCluster &b) {
12707 return a.Prob != b.Prob ?
12708 a.Prob > b.Prob :
12709 a.Low->getValue().slt(b.Low->getValue());
12710 });
12711
12712 // Rearrange the case blocks so that the last one falls through if possible
12713 // without changing the order of probabilities.
12714 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12715 --I;
12716 if (I->Prob > W.LastCluster->Prob)
12717 break;
12718 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12719 std::swap(*I, *W.LastCluster);
12720 break;
12721 }
12722 }
12723 }
12724
12725 // Compute total probability.
12726 BranchProbability DefaultProb = W.DefaultProb;
12727 BranchProbability UnhandledProbs = DefaultProb;
12728 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12729 UnhandledProbs += I->Prob;
12730
12731 MachineBasicBlock *CurMBB = W.MBB;
12732 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12733 bool FallthroughUnreachable = false;
12734 MachineBasicBlock *Fallthrough;
12735 if (I == W.LastCluster) {
12736 // For the last cluster, fall through to the default destination.
12737 Fallthrough = DefaultMBB;
12738 FallthroughUnreachable =
12740 } else {
12741 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
12742 CurMF->insert(BBI, Fallthrough);
12743 // Put Cond in a virtual register to make it available from the new blocks.
12745 }
12746 UnhandledProbs -= I->Prob;
12747
12748 switch (I->Kind) {
12749 case CC_JumpTable: {
12750 // FIXME: Optimize away range check based on pivot comparisons.
12751 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12752 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12753
12754 // The jump block hasn't been inserted yet; insert it here.
12755 MachineBasicBlock *JumpMBB = JT->MBB;
12756 CurMF->insert(BBI, JumpMBB);
12757
12758 auto JumpProb = I->Prob;
12759 auto FallthroughProb = UnhandledProbs;
12760
12761 // If the default statement is a target of the jump table, we evenly
12762 // distribute the default probability to successors of CurMBB. Also
12763 // update the probability on the edge from JumpMBB to Fallthrough.
12764 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12765 SE = JumpMBB->succ_end();
12766 SI != SE; ++SI) {
12767 if (*SI == DefaultMBB) {
12768 JumpProb += DefaultProb / 2;
12769 FallthroughProb -= DefaultProb / 2;
12770 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
12771 JumpMBB->normalizeSuccProbs();
12772 break;
12773 }
12774 }
12775
12776 // If the default clause is unreachable, propagate that knowledge into
12777 // JTH->FallthroughUnreachable which will use it to suppress the range
12778 // check.
12779 //
12780 // However, don't do this if we're doing branch target enforcement
12781 // (e.g. Arm BTI, or the cf-protection-branch module flag used by
12782 // RISC-V Zicfilp and X86 IBT), because a table branch _without_ a
12783 // range check can be a tempting JOP gadget - out-of-bounds inputs
12784 // that are impossible in correct execution become possible again if
12785 // an attacker can influence the control flow. So if an attacker
12786 // doesn't already have a branch target enforcement bypass available,
12787 // we don't want them to be able to get one out of this table branch.
12788 if (FallthroughUnreachable) {
12789 Function &CurFunc = CurMF->getFunction();
12790 bool HasCFBranch = false;
12791 if (auto *CFBranch = mdconst::extract_or_null<ConstantInt>(
12792 CurFunc.getParent()->getModuleFlag("cf-protection-branch")))
12793 HasCFBranch = !CFBranch->isZero();
12794 if (!HasCFBranch &&
12795 !CurFunc.hasFnAttribute("branch-target-enforcement"))
12796 JTH->FallthroughUnreachable = true;
12797 }
12798
12799 if (!JTH->FallthroughUnreachable)
12800 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12801 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12802 CurMBB->normalizeSuccProbs();
12803
12804 // The jump table header will be inserted in our current block, do the
12805 // range check, and fall through to our fallthrough block.
12806 JTH->HeaderBB = CurMBB;
12807 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12808
12809 // If we're in the right place, emit the jump table header right now.
12810 if (CurMBB == SwitchMBB) {
12811 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
12812 JTH->Emitted = true;
12813 }
12814 break;
12815 }
12816 case CC_BitTests: {
12817 // FIXME: Optimize away range check based on pivot comparisons.
12818 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12819
12820 // The bit test blocks haven't been inserted yet; insert them here.
12821 for (BitTestCase &BTC : BTB->Cases)
12822 CurMF->insert(BBI, BTC.ThisBB);
12823
12824 // Fill in fields of the BitTestBlock.
12825 BTB->Parent = CurMBB;
12826 BTB->Default = Fallthrough;
12827
12828 BTB->DefaultProb = UnhandledProbs;
12829 // If the cases in bit test don't form a contiguous range, we evenly
12830 // distribute the probability on the edge to Fallthrough to two
12831 // successors of CurMBB.
12832 if (!BTB->ContiguousRange) {
12833 BTB->Prob += DefaultProb / 2;
12834 BTB->DefaultProb -= DefaultProb / 2;
12835 }
12836
12837 if (FallthroughUnreachable)
12838 BTB->FallthroughUnreachable = true;
12839
12840 // If we're in the right place, emit the bit test header right now.
12841 if (CurMBB == SwitchMBB) {
12842 visitBitTestHeader(*BTB, SwitchMBB);
12843 BTB->Emitted = true;
12844 }
12845 break;
12846 }
12847 case CC_Range: {
12848 const Value *RHS, *LHS, *MHS;
12850 if (I->Low == I->High) {
12851 // Check Cond == I->Low.
12852 CC = ISD::SETEQ;
12853 LHS = Cond;
12854 RHS=I->Low;
12855 MHS = nullptr;
12856 } else {
12857 // Check I->Low <= Cond <= I->High.
12858 CC = ISD::SETLE;
12859 LHS = I->Low;
12860 MHS = Cond;
12861 RHS = I->High;
12862 }
12863
12864 // If Fallthrough is unreachable, fold away the comparison.
12865 if (FallthroughUnreachable)
12866 CC = ISD::SETTRUE;
12867
12868 // The false probability is the sum of all unhandled cases.
12869 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12870 getCurSDLoc(), I->Prob, UnhandledProbs);
12871
12872 if (CurMBB == SwitchMBB)
12873 visitSwitchCase(CB, SwitchMBB);
12874 else
12875 SL->SwitchCases.push_back(CB);
12876
12877 break;
12878 }
12879 }
12880 CurMBB = Fallthrough;
12881 }
12882}
12883
12884void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12885 const SwitchWorkListItem &W,
12886 Value *Cond,
12887 MachineBasicBlock *SwitchMBB) {
12888 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12889 "Clusters not sorted?");
12890 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12891
12892 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12893 SL->computeSplitWorkItemInfo(W);
12894
12895 // Use the first element on the right as pivot since we will make less-than
12896 // comparisons against it.
12897 CaseClusterIt PivotCluster = FirstRight;
12898 assert(PivotCluster > W.FirstCluster);
12899 assert(PivotCluster <= W.LastCluster);
12900
12901 CaseClusterIt FirstLeft = W.FirstCluster;
12902 CaseClusterIt LastRight = W.LastCluster;
12903
12904 const ConstantInt *Pivot = PivotCluster->Low;
12905
12906 // New blocks will be inserted immediately after the current one.
12908 ++BBI;
12909
12910 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12911 // we can branch to its destination directly if it's squeezed exactly in
12912 // between the known lower bound and Pivot - 1.
12913 MachineBasicBlock *LeftMBB;
12914 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12915 FirstLeft->Low == W.GE &&
12916 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12917 LeftMBB = FirstLeft->MBB;
12918 } else {
12919 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12920 FuncInfo.MF->insert(BBI, LeftMBB);
12921 WorkList.push_back(
12922 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
12923 // Put Cond in a virtual register to make it available from the new blocks.
12925 }
12926
12927 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12928 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12929 // directly if RHS.High equals the current upper bound.
12930 MachineBasicBlock *RightMBB;
12931 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12932 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12933 RightMBB = FirstRight->MBB;
12934 } else {
12935 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12936 FuncInfo.MF->insert(BBI, RightMBB);
12937 WorkList.push_back(
12938 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
12939 // Put Cond in a virtual register to make it available from the new blocks.
12941 }
12942
12943 // Create the CaseBlock record that will be used to lower the branch.
12944 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12945 getCurSDLoc(), LeftProb, RightProb);
12946
12947 if (W.MBB == SwitchMBB)
12948 visitSwitchCase(CB, SwitchMBB);
12949 else
12950 SL->SwitchCases.push_back(CB);
12951}
12952
12953// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12954// from the swith statement.
12956 BranchProbability PeeledCaseProb) {
12957 if (PeeledCaseProb == BranchProbability::getOne())
12959 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12960
12961 uint32_t Numerator = CaseProb.getNumerator();
12962 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
12963 return BranchProbability(Numerator, std::max(Numerator, Denominator));
12964}
12965
12966// Try to peel the top probability case if it exceeds the threshold.
12967// Return current MachineBasicBlock for the switch statement if the peeling
12968// does not occur.
12969// If the peeling is performed, return the newly created MachineBasicBlock
12970// for the peeled switch statement. Also update Clusters to remove the peeled
12971// case. PeeledCaseProb is the BranchProbability for the peeled case.
12972MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12973 const SwitchInst &SI, CaseClusterVector &Clusters,
12974 BranchProbability &PeeledCaseProb) {
12975 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12976 // Don't perform if there is only one cluster or optimizing for size.
12977 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12978 TM.getOptLevel() == CodeGenOptLevel::None ||
12979 SwitchMBB->getParent()->getFunction().hasMinSize())
12980 return SwitchMBB;
12981
12982 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
12983 unsigned PeeledCaseIndex = 0;
12984 bool SwitchPeeled = false;
12985 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12986 CaseCluster &CC = Clusters[Index];
12987 if (CC.Prob < TopCaseProb)
12988 continue;
12989 TopCaseProb = CC.Prob;
12990 PeeledCaseIndex = Index;
12991 SwitchPeeled = true;
12992 }
12993 if (!SwitchPeeled)
12994 return SwitchMBB;
12995
12996 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12997 << TopCaseProb << "\n");
12998
12999 // Record the MBB for the peeled switch statement.
13000 MachineFunction::iterator BBI(SwitchMBB);
13001 ++BBI;
13002 MachineBasicBlock *PeeledSwitchMBB =
13003 FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
13004 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
13005
13006 ExportFromCurrentBlock(SI.getCondition());
13007 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
13008 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
13009 nullptr, nullptr, TopCaseProb.getCompl()};
13010 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
13011
13012 Clusters.erase(PeeledCaseIt);
13013 for (CaseCluster &CC : Clusters) {
13014 LLVM_DEBUG(
13015 dbgs() << "Scale the probablity for one cluster, before scaling: "
13016 << CC.Prob << "\n");
13017 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
13018 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
13019 }
13020 PeeledCaseProb = TopCaseProb;
13021 return PeeledSwitchMBB;
13022}
13023
13024void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
13025 // Extract cases from the switch.
13026 BranchProbabilityInfo *BPI = FuncInfo.BPI;
13027 CaseClusterVector Clusters;
13028 Clusters.reserve(SI.getNumCases());
13029 for (auto I : SI.cases()) {
13030 MachineBasicBlock *Succ = FuncInfo.getMBB(I.getCaseSuccessor());
13031 const ConstantInt *CaseVal = I.getCaseValue();
13032 BranchProbability Prob =
13033 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
13034 : BranchProbability(1, SI.getNumCases() + 1);
13035 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
13036 }
13037
13038 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(SI.getDefaultDest());
13039
13040 // Cluster adjacent cases with the same destination. We do this at all
13041 // optimization levels because it's cheap to do and will make codegen faster
13042 // if there are many clusters.
13043 sortAndRangeify(Clusters);
13044
13045 // The branch probablity of the peeled case.
13046 BranchProbability PeeledCaseProb = BranchProbability::getZero();
13047 MachineBasicBlock *PeeledSwitchMBB =
13048 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
13049
13050 // If there is only the default destination, jump there directly.
13051 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
13052 if (Clusters.empty()) {
13053 assert(PeeledSwitchMBB == SwitchMBB);
13054 SwitchMBB->addSuccessor(DefaultMBB);
13055 if (DefaultMBB != NextBlock(SwitchMBB)) {
13056 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
13057 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
13058 }
13059 return;
13060 }
13061
13062 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
13063 DAG.getBFI());
13064 SL->findBitTestClusters(Clusters, &SI);
13065
13066 LLVM_DEBUG({
13067 dbgs() << "Case clusters: ";
13068 for (const CaseCluster &C : Clusters) {
13069 if (C.Kind == CC_JumpTable)
13070 dbgs() << "JT:";
13071 if (C.Kind == CC_BitTests)
13072 dbgs() << "BT:";
13073
13074 C.Low->getValue().print(dbgs(), true);
13075 if (C.Low != C.High) {
13076 dbgs() << '-';
13077 C.High->getValue().print(dbgs(), true);
13078 }
13079 dbgs() << ' ';
13080 }
13081 dbgs() << '\n';
13082 });
13083
13084 assert(!Clusters.empty());
13085 SwitchWorkList WorkList;
13086 CaseClusterIt First = Clusters.begin();
13087 CaseClusterIt Last = Clusters.end() - 1;
13088 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
13089 // Scale the branchprobability for DefaultMBB if the peel occurs and
13090 // DefaultMBB is not replaced.
13091 if (PeeledCaseProb != BranchProbability::getZero() &&
13092 DefaultMBB == FuncInfo.getMBB(SI.getDefaultDest()))
13093 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
13094 WorkList.push_back(
13095 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
13096
13097 while (!WorkList.empty()) {
13098 SwitchWorkListItem W = WorkList.pop_back_val();
13099 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
13100
13101 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
13102 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
13103 // For optimized builds, lower large range as a balanced binary tree.
13104 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
13105 continue;
13106 }
13107
13108 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
13109 }
13110}
13111
13112void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
13113 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13114 auto DL = getCurSDLoc();
13115 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13116 setValue(&I, DAG.getStepVector(DL, ResultVT));
13117}
13118
13119void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
13120 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13121 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13122
13123 SDLoc DL = getCurSDLoc();
13124 SDValue V = getValue(I.getOperand(0));
13125 assert(VT == V.getValueType() && "Malformed vector.reverse!");
13126
13127 if (VT.isScalableVector()) {
13128 setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
13129 return;
13130 }
13131
13132 // Use VECTOR_SHUFFLE for the fixed-length vector
13133 // to maintain existing behavior.
13134 SmallVector<int, 8> Mask;
13135 unsigned NumElts = VT.getVectorMinNumElements();
13136 for (unsigned i = 0; i != NumElts; ++i)
13137 Mask.push_back(NumElts - 1 - i);
13138
13139 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
13140}
13141
13142void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
13143 unsigned Factor) {
13144 auto DL = getCurSDLoc();
13145 SDValue InVec = getValue(I.getOperand(0));
13146
13147 SmallVector<EVT, 4> ValueVTs;
13148 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
13149 ValueVTs);
13150
13151 EVT OutVT = ValueVTs[0];
13152 unsigned OutNumElts = OutVT.getVectorMinNumElements();
13153
13154 SmallVector<SDValue, 4> SubVecs(Factor);
13155 for (unsigned i = 0; i != Factor; ++i) {
13156 assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
13157 SubVecs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
13158 DAG.getVectorIdxConstant(OutNumElts * i, DL));
13159 }
13160
13161 SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
13162 DAG.getVTList(ValueVTs), SubVecs);
13163 setValue(&I, Res);
13164}
13165
13166void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
13167 unsigned Factor) {
13168 auto DL = getCurSDLoc();
13169 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13170 EVT InVT = getValue(I.getOperand(0)).getValueType();
13171 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13172
13173 SmallVector<SDValue, 8> InVecs(Factor);
13174 for (unsigned i = 0; i < Factor; ++i) {
13175 InVecs[i] = getValue(I.getOperand(i));
13176 assert(InVecs[i].getValueType() == InVecs[0].getValueType() &&
13177 "Expected VTs to be the same");
13178 }
13179
13180 SmallVector<EVT, 8> ValueVTs(Factor, InVT);
13181 SDValue Res =
13182 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList(ValueVTs), InVecs);
13183
13185 for (unsigned i = 0; i < Factor; ++i)
13186 Results[i] = Res.getValue(i);
13187
13188 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Results);
13189 setValue(&I, Res);
13190}
13191
13192void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
13193 SmallVector<EVT, 4> ValueVTs;
13194 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
13195 ValueVTs);
13196 unsigned NumValues = ValueVTs.size();
13197 if (NumValues == 0) return;
13198
13200 SDValue Op = getValue(I.getOperand(0));
13201
13202 for (unsigned i = 0; i != NumValues; ++i)
13203 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
13204 SDValue(Op.getNode(), Op.getResNo() + i));
13205
13207 DAG.getVTList(ValueVTs), Values));
13208}
13209
13210void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
13211 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13212 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
13213
13214 SDLoc DL = getCurSDLoc();
13215 SDValue V1 = getValue(I.getOperand(0));
13216 SDValue V2 = getValue(I.getOperand(1));
13217 const bool IsLeft = I.getIntrinsicID() == Intrinsic::vector_splice_left;
13218
13219 // VECTOR_SHUFFLE doesn't support a scalable or non-constant mask.
13220 if (VT.isScalableVector() || !isa<ConstantInt>(I.getOperand(2))) {
13221 SDValue Offset = DAG.getZExtOrTrunc(
13222 getValue(I.getOperand(2)), DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
13223 setValue(&I, DAG.getNode(IsLeft ? ISD::VECTOR_SPLICE_LEFT
13225 DL, VT, V1, V2, Offset));
13226 return;
13227 }
13228 uint64_t Imm = cast<ConstantInt>(I.getOperand(2))->getZExtValue();
13229
13230 unsigned NumElts = VT.getVectorNumElements();
13231
13232 uint64_t Idx = IsLeft ? Imm : NumElts - Imm;
13233
13234 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
13235 SmallVector<int, 8> Mask;
13236 for (unsigned i = 0; i < NumElts; ++i)
13237 Mask.push_back(Idx + i);
13238 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
13239}
13240
13241// Consider the following MIR after SelectionDAG, which produces output in
13242// phyregs in the first case or virtregs in the second case.
13243//
13244// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
13245// %5:gr32 = COPY $ebx
13246// %6:gr32 = COPY $edx
13247// %1:gr32 = COPY %6:gr32
13248// %0:gr32 = COPY %5:gr32
13249//
13250// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
13251// %1:gr32 = COPY %6:gr32
13252// %0:gr32 = COPY %5:gr32
13253//
13254// Given %0, we'd like to return $ebx in the first case and %5 in the second.
13255// Given %1, we'd like to return $edx in the first case and %6 in the second.
13256//
13257// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
13258// to a single virtreg (such as %0). The remaining outputs monotonically
13259// increase in virtreg number from there. If a callbr has no outputs, then it
13260// should not have a corresponding callbr landingpad; in fact, the callbr
13261// landingpad would not even be able to refer to such a callbr.
13263 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
13264 // There is definitely at least one copy.
13265 assert(MI->getOpcode() == TargetOpcode::COPY &&
13266 "start of copy chain MUST be COPY");
13267 Reg = MI->getOperand(1).getReg();
13268
13269 // If the copied register in the first copy must be virtual.
13270 assert(Reg.isVirtual() && "expected COPY of virtual register");
13271 MI = MRI.def_begin(Reg)->getParent();
13272
13273 // There may be an optional second copy.
13274 if (MI->getOpcode() == TargetOpcode::COPY) {
13275 assert(Reg.isVirtual() && "expected COPY of virtual register");
13276 Reg = MI->getOperand(1).getReg();
13277 assert(Reg.isPhysical() && "expected COPY of physical register");
13278 } else {
13279 // The start of the chain must be an INLINEASM_BR.
13280 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13281 "end of copy chain MUST be INLINEASM_BR");
13282 }
13283
13284 return Reg;
13285}
13286
13287// We must do this walk rather than the simpler
13288// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
13289// otherwise we will end up with copies of virtregs only valid along direct
13290// edges.
13291void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
13292 SmallVector<EVT, 8> ResultVTs;
13293 SmallVector<SDValue, 8> ResultValues;
13294 const auto *CBR =
13295 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
13296
13297 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13298 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
13299 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
13300
13301 Register InitialDef = FuncInfo.ValueMap[CBR];
13302 SDValue Chain = DAG.getRoot();
13303
13304 // Re-parse the asm constraints string.
13305 TargetLowering::AsmOperandInfoVector TargetConstraints =
13306 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
13307 for (auto &T : TargetConstraints) {
13308 SDISelAsmOperandInfo OpInfo(T);
13309 if (OpInfo.Type != InlineAsm::isOutput)
13310 continue;
13311
13312 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
13313 // individual constraint.
13314 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
13315
13316 switch (OpInfo.ConstraintType) {
13319 // Fill in OpInfo.AssignedRegs.Regs.
13320 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
13321
13322 // getRegistersForValue may produce 1 to many registers based on whether
13323 // the OpInfo.ConstraintVT is legal on the target or not.
13324 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
13325 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
13326 if (OriginalDef.isPhysical())
13327 FuncInfo.MBB->addLiveIn(OriginalDef);
13328 // Update the assigned registers to use the original defs.
13329 Reg = OriginalDef;
13330 }
13331
13332 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
13333 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
13334 ResultValues.push_back(V);
13335 ResultVTs.push_back(OpInfo.ConstraintVT);
13336 break;
13337 }
13339 SDValue Flag;
13341 OpInfo, DAG);
13342 ++InitialDef;
13343 ResultValues.push_back(V);
13344 ResultVTs.push_back(OpInfo.ConstraintVT);
13345 break;
13346 }
13347 default:
13348 break;
13349 }
13350 }
13352 DAG.getVTList(ResultVTs), ResultValues);
13353 setValue(&I, V);
13354}
unsigned NumParts
return SDValue()
static unsigned getIntrinsicID(const SDNode *N)
bool IsScalable
const TargetLowering & TLI
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
dxil translate DXIL Translate Metadata
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition FastISel.cpp:947
#define Check(C,...)
static Value * getCondition(Instruction *I)
Hexagon Common GEP
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
static void computeConstraintToUse(const TargetLowering *TLI, TargetLowering::AsmOperandInfo &OpInfo)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
static bool isUndef(const MachineInstr &MI)
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file contains the declarations for metadata subclasses.
Type::TypeID TypeID
#define T
#define T1
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
#define P(N)
if(PassOpts->AAPipeline)
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
MachineInstr *& Lo
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static void failForInvalidBundles(const CallBase &I, StringRef Name, ArrayRef< uint32_t > AllowedBundles)
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, SelectionDAG &DAG)
Prepare DAG-level operands.
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static const GlobalValue * getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx, const MachineFunction &MF)
If V is the address of a describable global, possibly displaced by a constant, return the global and ...
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
DenseMap< const Argument *, std::pair< const AllocaInst *, const StoreInst * > > ArgCopyElisionMapTy
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< Register, TypeSize > > &Regs, const SDValue &N)
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static bool determineConstraints(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, const TargetMachine &TM, SelectionDAG &DAG, const BasicBlock *EHPadBB)
DetermineConstraints - Find the constraints to use for inline asm operands.
static bool constructOperandInfo(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, SelectionDAGBuilder &Builder, const TargetLowering &TLI, ExtraFlags &ExtraInfo)
Construct operand info objects.
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts, MVT RegisterVT, MVT PartVT)
Emit a fatal error if the broken down registers don't match part type or count.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static FPClassTest getNoFPClass(const Instruction &I)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const uint8_t *MatcherTable, size_t &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
TTI::PartialReductionExtendKind ExtendKind
uint16_t RegSizeInBits(const MCRegisterInfo &MRI, MCRegister RegNo)
Value * RHS
Value * LHS
The Input class is used to parse a yaml document into in-memory structs and vectors.
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static LLVM_ABI Semantics SemanticsToEnum(const llvm::fltSemantics &Sem)
Definition APFloat.cpp:185
static LLVM_ABI const fltSemantics * getArbitraryFPSemantics(StringRef Format)
Returns the fltSemantics for a given arbitrary FP format string, or nullptr if invalid.
Definition APFloat.cpp:6340
Class for arbitrary precision integers.
Definition APInt.h:78
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:330
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition Function.cpp:332
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition Argument.h:50
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
iterator end() const
Definition ArrayRef.h:130
size_t size() const
Get the array size.
Definition ArrayRef.h:141
iterator begin() const
Definition ArrayRef.h:129
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:448
LLVM Basic Block Representation.
Definition BasicBlock.h:62
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
InstListType::const_iterator const_iterator
Definition BasicBlock.h:171
LLVM_ABI bool isEntryBlock() const
Return true if this is the entry block of the containing function.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
This class represents a no-op cast from one type to another.
The address of a basic block.
Definition Constants.h:1088
Analysis providing branch probability information.
LLVM_ABI BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
LLVM_ABI bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static constexpr BranchProbability getOne()
static uint32_t getDenominator()
static constexpr BranchProbability getUnknown()
static constexpr BranchProbability getZero()
uint32_t getNumerator() const
LLVM_ABI uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
LLVM_ABI bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition InstrTypes.h:728
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
Conditional Branch instruction.
Class for constant bytes.
Definition Constants.h:281
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition Constants.h:755
A constant value that is initialized with an expression using other constant values.
Definition Constants.h:1316
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
A signed pointer, in the ptrauth sense.
Definition Constants.h:1223
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition Constants.h:674
This is an important base class in LLVM.
Definition Constant.h:44
This is the common base class for constrained floating point intrinsics.
LLVM_ABI std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
LLVM_ABI unsigned getNonMetadataArgCount() const
DWARF expression.
LLVM_ABI bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static LLVM_ABI std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
LLVM_ABI uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static LLVM_ABI const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static LLVM_ABI DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
Base class for variables.
LLVM_ABI std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool isBigEndian() const
Definition DataLayout.h:221
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LLVM_ABI Value * getVariableLocationOp(unsigned OpIdx) const
DIExpression * getExpression() const
DILocalVariable * getVariable() const
LLVM_ABI iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
A debug info location.
Definition DebugLoc.h:126
LLVM_ABI DILocation * getInlinedAt() const
Definition DebugLoc.cpp:58
bool empty() const
Definition DenseMap.h:716
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT, true > const_iterator
Definition DenseMap.h:679
void reserve(size_type NumEntries)
Grow the DenseMap so that it can contain at least NumEntries items before resizing again.
Definition DenseMap.h:721
iterator end()
Definition DenseMap.h:686
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
Diagnostic information for inline asm reporting.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:312
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
Lightweight error class with error context and mandatory checking.
Definition Error.h:159
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition FastISel.h:67
bool allowReassoc() const
Flag queries.
Definition FMF.h:64
An instruction for ordering other memory operations.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
MachineBasicBlock * getMBB(const BasicBlock *BB) const
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
MachineBasicBlock * MBB
MBB - The current block.
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition Function.h:803
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:705
bool hasParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const
check if an attributes is in the list of attributes.
Definition Function.cpp:743
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:329
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition Function.h:252
size_t arg_size() const
Definition Function.h:895
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
Garbage collection metadata for a single function.
Definition GCMetadata.h:80
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
bool isInBounds() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
void setMemConstraint(ConstraintCode C)
setMemConstraint - Augment an existing flag with the constraint code for a memory constraint.
Definition InlineAsm.h:431
This instruction inserts a struct field of array element value into an aggregate value.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
A helper class to return the specified delimiter string after the first invocation of operator String...
An instruction for reading from memory.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
static LocationSize upperBound(uint64_t Value)
bool usesPerInvokeEHLabels(ExceptionHandling EH) const
Returns true if the exception tables reference the per-invoke EH labels emitted around invokes.
Definition MCAsmInfo.h:692
LLVM_ABI MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
LLVM_ABI MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
const MCAsmInfo & getAsmInfo() const
Definition MCContext.h:407
unsigned getID() const
getID() - Return the register class ID number.
bool isAllocatable() const
isAllocatable - Return true if this register class may be used to create virtual registers.
const MCPhysReg * iterator
iterator begin() const
begin/end - Return all of the registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
iterator end() const
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition MCSymbol.h:42
Metadata node.
Definition Metadata.h:1087
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1443
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:613
Machine Value Type.
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
std::string getString() const
Return the value type as a string, e.g. i32.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHContTarget(bool V=true)
Indicates if this is a target of Windows EH Continuation Guard.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
MachineInstrBundleIterator< MachineInstr > iterator
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
void setHasEHContTarget(bool V)
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI MCRegister getLiveInPhysReg(Register VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition MapVector.h:119
bool contains(const KeyT &Key) const
Definition MapVector.h:149
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:185
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Definition Module.cpp:377
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:882
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(Register VReg)
static SDDbgOperand fromConst(const Value *Const)
static SDDbgOperand fromGlobalAddr(const GlobalValue *GV)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const CondBrInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
DenseMap< const Constant *, Register > ConstantsOut
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
SDValue lowerStartEH(SDValue Chain, const BasicBlock *EHPadBB, MCSymbol *&BeginLabel)
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
const TargetTransformInfo * TTI
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
SDValue lowerNoFPClassToAssertNoFPClass(SelectionDAG &DAG, const Instruction &I, SDValue Op)
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void setValueToPoison(const Value *V, const SDLoc &dl)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, Register Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool canTailCall(const CallBase &CB) const
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void CopyValueToVirtualRegister(const Value *V, Register Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, const TargetLibraryInfo *li, const TargetTransformInfo &TTI)
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
SDValue getFPOperationRoot(fp::ExceptionBehavior EB)
Return the current virtual root of the Selection DAG, flushing PendingConstrainedFP or PendingConstra...
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineRegisterInfo * RegInfo
std::unique_ptr< SwiftErrorValueTracking > SwiftError
virtual void emitFunctionEntryCode()
std::unique_ptr< SelectionDAGBuilder > SDB
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemccpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue C, SDValue Size, const CallInst *CI) const
Emit target-specific code that performs a memccpy, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, const CallInst *CI) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrstr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, const CallInst *CI) const
Emit target-specific code that performs a strstr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo, const CallInst *CI) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, const CallInst *CI) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy, const CallInst *CI) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI bool shouldOptForSize() const
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
LLVM_ABI void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
LLVM_ABI SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void resize(size_type N)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitFunctionBasedCheckStackProtector() const
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
Multiway switch.
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Provides information about what library functions are available for the current target.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Returns the desired alignment for ByVal or InAlloca aggregate function arguments in the caller parame...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
virtual bool isAtomicAlignmentSupported(Align Alignment, uint64_t SizeInBytes) const
Return true if the target supports an atomic access of SizeInBytes bytes at the given Alignment.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
virtual Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual bool useStackGuardMixFP() const
If this function returns true, stack protection checks should mix the frame pointer (or whichever poi...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr, CodeGenOptLevel OptLevel=CodeGenOptLevel::Default) const
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
MachineMemOperand::Flags getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
virtual Value * getSDagStackGuard(const Module &M, const LibcallLoweringInfo &Libcalls) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
virtual Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_UDIV, ISD::VP_SDIV,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual SDValue emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
Definition Type.cpp:170
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:267
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
bool isTokenTy() const
Return true if this is 'token'.
Definition Type.h:231
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
Unconditional Branch instruction.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_iterator op_begin()
Definition User.h:259
unsigned getNumOperands() const
Definition User.h:229
op_iterator op_end()
Definition User.h:261
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
This is the common base class for vector predication intrinsics.
static LLVM_ABI std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
LLVM_ABI MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:707
bool use_empty() const
Definition Value.h:348
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
const ParentTy * getParent() const
Definition ilist_node.h:34
A raw_ostream that writes to an std::string.
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ CONVERGENCECTRL_ANCHOR
The llvm.experimental.convergence.* intrinsics.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition ISDOpcodes.h:47
@ SET_FPENV
Sets the current floating-point environment.
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ COND_LOOP
COND_LOOP is a conditional branch to self, used for implementing efficient conditional traps.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition ISDOpcodes.h:170
@ VECREDUCE_FMINIMUMNUM
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ STACKADDRESS
STACKADDRESS - Represents the llvm.stackaddress intrinsic.
Definition ISDOpcodes.h:129
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:397
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:403
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition ISDOpcodes.h:174
@ GlobalAddress
Definition ISDOpcodes.h:90
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
Definition ISDOpcodes.h:665
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
Definition ISDOpcodes.h:533
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
Definition ISDOpcodes.h:520
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:410
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition ISDOpcodes.h:158
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ SET_ROUNDING
Set rounding mode.
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition ISDOpcodes.h:119
@ CONVERGENCECTRL_ENTRY
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PARTIAL_REDUCE_FMLA
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:640
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
Definition ISDOpcodes.h:80
@ PtrAuthGlobalAddress
A ptrauth constant.
Definition ISDOpcodes.h:102
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition ISDOpcodes.h:50
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition ISDOpcodes.h:141
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition ISDOpcodes.h:137
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition ISDOpcodes.h:389
@ PATCHPOINT
The llvm.experimental.patchpoint.
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
Definition ISDOpcodes.h:669
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition ISDOpcodes.h:650
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:416
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition ISDOpcodes.h:152
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:112
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
Definition ISDOpcodes.h:807
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition ISDOpcodes.h:505
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ RELOC_NONE
Issue a no-op relocation against a given symbol at the current location.
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition ISDOpcodes.h:759
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
Definition ISDOpcodes.h:673
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition ISDOpcodes.h:430
@ STACKMAP
The llvm.experimental.stackmap intrinsic.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition ISDOpcodes.h:715
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition ISDOpcodes.h:124
@ CLEAR_CACHE
llvm.clear_cache intrinsic Operands: Input Chain, Start Addres, End Address Outputs: Output Chain
@ CONVERGENCECTRL_LOOP
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ VECREDUCE_FMINIMUM
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition ISDOpcodes.h:164
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ BRCOND
BRCOND - Conditional branch.
@ VECREDUCE_SEQ_FMUL
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
Definition ISDOpcodes.h:645
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:629
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ CTTZ_ELTS_ZERO_POISON
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
Definition ISDOpcodes.h:775
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Flag
These should be considered private to the implementation of the MCInstrDesc class.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_VScale()
Matches a call to llvm.vscale().
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
Offsets
Offsets in bytes from the start of the input buffer.
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
LLVM_ABI void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
std::vector< CaseCluster > CaseClusterVector
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
CaseClusterVector::iterator CaseClusterIt
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition Dwarf.h:149
ExceptionBehavior
Exception behavior used for floating point operations.
Definition FPEnv.h:39
@ ebStrict
This corresponds to "fpexcept.strict".
Definition FPEnv.h:42
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition FPEnv.h:41
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition FPEnv.h:40
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)
Extract a Value from Metadata, allowing null.
Definition Metadata.h:695
constexpr float log2ef
Definition MathExtras.h:52
constexpr double e
constexpr float ln2f
Definition MathExtras.h:50
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:339
LLVM_ABI ISD::NodeType getAtomicRMWOpcode(AtomicRMWInst::BinOp Op)
Return the atomic node opcode for an atomicrmw operation.
Definition Analysis.cpp:292
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
LLVM_ABI bool canDescribeGlobalAddressInLocationList(const MachineFunction &MF)
Test if the debug info for MF can name a describable global address in a location list too,...
Definition Analysis.cpp:695
LLVM_ABI ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition Analysis.cpp:248
LLVM_ABI cl::opt< bool > ProfcheckDisableMetadataFixes
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
SDValue peekThroughFreeze(SDValue V)
Return the non-frozen source operand of V if it exists.
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
RelativeUniformCounterPtr Values
Definition InstrProf.h:91
@ Unknown
Not known to have no common set bits.
LLVM_ABI void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs=nullptr, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition Analysis.cpp:121
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2575
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Done
Definition Threading.h:60
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
LLVM_ABI void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
LLVM_ABI bool isExceptionPointerAndSelectorType(Type *Ty)
Return true if landingpad result type Ty is a struct of an exception pointer (pointer or integer) and...
Definition Analysis.cpp:201
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
static ConstantRange getRange(Value *Op, SCCPSolver &Solver, const SmallPtrSetImpl< Value * > &InsertedValues)
Helper for getting ranges from Solver.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2229
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
auto cast_or_null(const Y &Val)
Definition Casting.h:714
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
gep_type_iterator gep_type_end(const User *GEP)
LLVM_ABI LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
Definition STLExtras.h:2189
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
Definition STLExtras.h:1167
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
LLVM_ABI void ComputeValueTypes(const DataLayout &DL, Type *Ty, SmallVectorImpl< Type * > &Types, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
Given an LLVM IR type, compute non-aggregate subtypes.
Definition Analysis.cpp:74
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
FPOp
The floating-point operations supported by the folding helpers.
Definition FPFold.h:25
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition STLExtras.h:869
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
generic_gep_type_iterator<> gep_type_iterator
auto succ_size(const MachineBasicBlock *BB)
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition STLExtras.h:300
LLVM_ABI ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition Analysis.cpp:214
LLVM_ABI MDNode * getMemCacheHint(const Instruction &I, unsigned OperandNo=0)
Return the hint payload for operand OperandNo of I, or nullptr if absent.
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition Local.cpp:2353
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
LLVM_ABI bool isOnlyUsedInZeroEqualityComparison(const Instruction *CtxI)
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
LLVM_ABI bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition Analysis.cpp:709
DWARFExpression::Operation Op
@ Dynamic
Denotes mode unknown at compile time.
LLVM_ABI ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition Analysis.cpp:236
LLVM_ABI const GlobalValue * getDescribableGlobalAddress(const Constant *C, int64_t &Offset, const MachineFunction &MF)
If C is the address of a global, possibly displaced by a constant, return that global and set Offset ...
Definition Analysis.cpp:676
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
LLVM_ABI std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition FPEnv.cpp:25
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2210
LLVM_ABI GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition Analysis.cpp:183
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
LLVM_ABI Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
LLVM_ABI unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition Analysis.cpp:35
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:347
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
#define NC
Definition regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
uint64_t getScalarStoreSize() const
Definition ValueTypes.h:440
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:98
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
Definition ValueTypes.h:212
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isFixedLengthVector() const
Definition ValueTypes.h:214
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:330
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
Definition ValueTypes.h:136
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition InlineAsm.h:129
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:262
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
A lightweight accessor for an operand bundle meant to be passed around by value.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< std::pair< Register, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< Register, 4 > Regs
This list holds the registers assigned to the values.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
void setUnpredictable(bool b)
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition MapVector.h:344
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
Register Reg
The virtual register containing the index of the jump table entry to jump to.
MachineBasicBlock * Default
The MBB of the default bb, which is a successor of the range check MBB.
unsigned JTI
The JumpTableIndex for this jump table in the function.
MachineBasicBlock * MBB
The MBB into which to emit the code for the indirect jump.
std::optional< SDLoc > SL
The debug location of the instruction this JumpTable was produced from.
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setDeactivationSymbol(GlobalValue *Sym)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
Type * OrigRetTy
Original unlegalized return type.
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
LLVM_ABI void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)