LLVM 20.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
58#include "llvm/IR/Argument.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/BasicBlock.h"
61#include "llvm/IR/CFG.h"
62#include "llvm/IR/CallingConv.h"
63#include "llvm/IR/Constant.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfo.h"
72#include "llvm/IR/Function.h"
74#include "llvm/IR/InlineAsm.h"
75#include "llvm/IR/InstrTypes.h"
78#include "llvm/IR/Intrinsics.h"
79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsWebAssembly.h"
82#include "llvm/IR/LLVMContext.h"
84#include "llvm/IR/Metadata.h"
85#include "llvm/IR/Module.h"
86#include "llvm/IR/Operator.h"
88#include "llvm/IR/Statepoint.h"
89#include "llvm/IR/Type.h"
90#include "llvm/IR/User.h"
91#include "llvm/IR/Value.h"
92#include "llvm/MC/MCContext.h"
97#include "llvm/Support/Debug.h"
106#include <cstddef>
107#include <deque>
108#include <iterator>
109#include <limits>
110#include <optional>
111#include <tuple>
112
113using namespace llvm;
114using namespace PatternMatch;
115using namespace SwitchCG;
116
117#define DEBUG_TYPE "isel"
118
119/// LimitFloatPrecision - Generate low-precision inline sequences for
120/// some float libcalls (6, 8 or 12 bits).
121static unsigned LimitFloatPrecision;
122
123static cl::opt<bool>
124 InsertAssertAlign("insert-assert-align", cl::init(true),
125 cl::desc("Insert the experimental `assertalign` node."),
127
129 LimitFPPrecision("limit-float-precision",
130 cl::desc("Generate low-precision inline sequences "
131 "for some float libcalls"),
133 cl::init(0));
134
136 "switch-peel-threshold", cl::Hidden, cl::init(66),
137 cl::desc("Set the case probability threshold for peeling the case from a "
138 "switch statement. A value greater than 100 will void this "
139 "optimization"));
140
141// Limit the width of DAG chains. This is important in general to prevent
142// DAG-based analysis from blowing up. For example, alias analysis and
143// load clustering may not complete in reasonable time. It is difficult to
144// recognize and avoid this situation within each individual analysis, and
145// future analyses are likely to have the same behavior. Limiting DAG width is
146// the safe approach and will be especially important with global DAGs.
147//
148// MaxParallelChains default is arbitrarily high to avoid affecting
149// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
150// sequence over this should have been converted to llvm.memcpy by the
151// frontend. It is easy to induce this behavior with .ll code such as:
152// %buffer = alloca [4096 x i8]
153// %data = load [4096 x i8]* %argPtr
154// store [4096 x i8] %data, [4096 x i8]* %buffer
155static const unsigned MaxParallelChains = 64;
156
158 const SDValue *Parts, unsigned NumParts,
159 MVT PartVT, EVT ValueVT, const Value *V,
160 SDValue InChain,
161 std::optional<CallingConv::ID> CC);
162
163/// getCopyFromParts - Create a value that contains the specified legal parts
164/// combined into the value they represent. If the parts combine to a type
165/// larger than ValueVT then AssertOp can be used to specify whether the extra
166/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
167/// (ISD::AssertSext).
168static SDValue
169getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
170 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
171 SDValue InChain,
172 std::optional<CallingConv::ID> CC = std::nullopt,
173 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
174 // Let the target assemble the parts if it wants to
175 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
176 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
177 PartVT, ValueVT, CC))
178 return Val;
179
180 if (ValueVT.isVector())
181 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
182 InChain, CC);
183
184 assert(NumParts > 0 && "No parts to assemble!");
185 SDValue Val = Parts[0];
186
187 if (NumParts > 1) {
188 // Assemble the value from multiple parts.
189 if (ValueVT.isInteger()) {
190 unsigned PartBits = PartVT.getSizeInBits();
191 unsigned ValueBits = ValueVT.getSizeInBits();
192
193 // Assemble the power of 2 part.
194 unsigned RoundParts = llvm::bit_floor(NumParts);
195 unsigned RoundBits = PartBits * RoundParts;
196 EVT RoundVT = RoundBits == ValueBits ?
197 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
198 SDValue Lo, Hi;
199
200 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
201
202 if (RoundParts > 2) {
203 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
204 InChain);
205 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
206 PartVT, HalfVT, V, InChain);
207 } else {
208 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
209 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
210 }
211
212 if (DAG.getDataLayout().isBigEndian())
213 std::swap(Lo, Hi);
214
215 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
216
217 if (RoundParts < NumParts) {
218 // Assemble the trailing non-power-of-2 part.
219 unsigned OddParts = NumParts - RoundParts;
220 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
221 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
222 OddVT, V, InChain, CC);
223
224 // Combine the round and odd parts.
225 Lo = Val;
226 if (DAG.getDataLayout().isBigEndian())
227 std::swap(Lo, Hi);
228 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
229 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
230 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
231 DAG.getConstant(Lo.getValueSizeInBits(), DL,
233 TotalVT, DAG.getDataLayout())));
234 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
235 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
236 }
237 } else if (PartVT.isFloatingPoint()) {
238 // FP split into multiple FP parts (for ppcf128)
239 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
240 "Unexpected split");
241 SDValue Lo, Hi;
242 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
243 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
244 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
245 std::swap(Lo, Hi);
246 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
247 } else {
248 // FP split into integer parts (soft fp)
249 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
250 !PartVT.isVector() && "Unexpected split");
251 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
252 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
253 InChain, CC);
254 }
255 }
256
257 // There is now one part, held in Val. Correct it to match ValueVT.
258 // PartEVT is the type of the register class that holds the value.
259 // ValueVT is the type of the inline asm operation.
260 EVT PartEVT = Val.getValueType();
261
262 if (PartEVT == ValueVT)
263 return Val;
264
265 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
266 ValueVT.bitsLT(PartEVT)) {
267 // For an FP value in an integer part, we need to truncate to the right
268 // width first.
269 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
270 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
271 }
272
273 // Handle types that have the same size.
274 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
275 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
276
277 // Handle types with different sizes.
278 if (PartEVT.isInteger() && ValueVT.isInteger()) {
279 if (ValueVT.bitsLT(PartEVT)) {
280 // For a truncate, see if we have any information to
281 // indicate whether the truncated bits will always be
282 // zero or sign-extension.
283 if (AssertOp)
284 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
285 DAG.getValueType(ValueVT));
286 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
287 }
288 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
289 }
290
291 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
292 // FP_ROUND's are always exact here.
293 if (ValueVT.bitsLT(Val.getValueType())) {
294
295 SDValue NoChange =
297
299 llvm::Attribute::StrictFP)) {
300 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
301 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
302 NoChange);
303 }
304
305 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
306 }
307
308 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
309 }
310
311 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
312 // then truncating.
313 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
314 ValueVT.bitsLT(PartEVT)) {
315 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
316 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
317 }
318
319 report_fatal_error("Unknown mismatch in getCopyFromParts!");
320}
321
323 const Twine &ErrMsg) {
324 const Instruction *I = dyn_cast_or_null<Instruction>(V);
325 if (!V)
326 return Ctx.emitError(ErrMsg);
327
328 const char *AsmError = ", possible invalid constraint for vector type";
329 if (const CallInst *CI = dyn_cast<CallInst>(I))
330 if (CI->isInlineAsm())
331 return Ctx.emitError(I, ErrMsg + AsmError);
332
333 return Ctx.emitError(I, ErrMsg);
334}
335
336/// getCopyFromPartsVector - Create a value that contains the specified legal
337/// parts combined into the value they represent. If the parts combine to a
338/// type larger than ValueVT then AssertOp can be used to specify whether the
339/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
340/// ValueVT (ISD::AssertSext).
342 const SDValue *Parts, unsigned NumParts,
343 MVT PartVT, EVT ValueVT, const Value *V,
344 SDValue InChain,
345 std::optional<CallingConv::ID> CallConv) {
346 assert(ValueVT.isVector() && "Not a vector value");
347 assert(NumParts > 0 && "No parts to assemble!");
348 const bool IsABIRegCopy = CallConv.has_value();
349
350 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
351 SDValue Val = Parts[0];
352
353 // Handle a multi-element vector.
354 if (NumParts > 1) {
355 EVT IntermediateVT;
356 MVT RegisterVT;
357 unsigned NumIntermediates;
358 unsigned NumRegs;
359
360 if (IsABIRegCopy) {
362 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
363 NumIntermediates, RegisterVT);
364 } else {
365 NumRegs =
366 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
367 NumIntermediates, RegisterVT);
368 }
369
370 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
371 NumParts = NumRegs; // Silence a compiler warning.
372 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
373 assert(RegisterVT.getSizeInBits() ==
374 Parts[0].getSimpleValueType().getSizeInBits() &&
375 "Part type sizes don't match!");
376
377 // Assemble the parts into intermediate operands.
378 SmallVector<SDValue, 8> Ops(NumIntermediates);
379 if (NumIntermediates == NumParts) {
380 // If the register was not expanded, truncate or copy the value,
381 // as appropriate.
382 for (unsigned i = 0; i != NumParts; ++i)
383 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
384 V, InChain, CallConv);
385 } else if (NumParts > 0) {
386 // If the intermediate type was expanded, build the intermediate
387 // operands from the parts.
388 assert(NumParts % NumIntermediates == 0 &&
389 "Must expand into a divisible number of parts!");
390 unsigned Factor = NumParts / NumIntermediates;
391 for (unsigned i = 0; i != NumIntermediates; ++i)
392 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
393 IntermediateVT, V, InChain, CallConv);
394 }
395
396 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
397 // intermediate operands.
398 EVT BuiltVectorTy =
399 IntermediateVT.isVector()
401 *DAG.getContext(), IntermediateVT.getScalarType(),
402 IntermediateVT.getVectorElementCount() * NumParts)
404 IntermediateVT.getScalarType(),
405 NumIntermediates);
406 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
408 DL, BuiltVectorTy, Ops);
409 }
410
411 // There is now one part, held in Val. Correct it to match ValueVT.
412 EVT PartEVT = Val.getValueType();
413
414 if (PartEVT == ValueVT)
415 return Val;
416
417 if (PartEVT.isVector()) {
418 // Vector/Vector bitcast.
419 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
420 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
421
422 // If the parts vector has more elements than the value vector, then we
423 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
424 // Extract the elements we want.
425 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
428 (PartEVT.getVectorElementCount().isScalable() ==
429 ValueVT.getVectorElementCount().isScalable()) &&
430 "Cannot narrow, it would be a lossy transformation");
431 PartEVT =
433 ValueVT.getVectorElementCount());
434 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
435 DAG.getVectorIdxConstant(0, DL));
436 if (PartEVT == ValueVT)
437 return Val;
438 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
439 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
440
441 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
442 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
443 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
444 }
445
446 // Promoted vector extract
447 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
448 }
449
450 // Trivial bitcast if the types are the same size and the destination
451 // vector type is legal.
452 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
453 TLI.isTypeLegal(ValueVT))
454 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
455
456 if (ValueVT.getVectorNumElements() != 1) {
457 // Certain ABIs require that vectors are passed as integers. For vectors
458 // are the same size, this is an obvious bitcast.
459 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
460 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
461 } else if (ValueVT.bitsLT(PartEVT)) {
462 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
463 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
464 // Drop the extra bits.
465 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
466 return DAG.getBitcast(ValueVT, Val);
467 }
468
470 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
471 return DAG.getUNDEF(ValueVT);
472 }
473
474 // Handle cases such as i8 -> <1 x i1>
475 EVT ValueSVT = ValueVT.getVectorElementType();
476 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
477 unsigned ValueSize = ValueSVT.getSizeInBits();
478 if (ValueSize == PartEVT.getSizeInBits()) {
479 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
480 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
481 // It's possible a scalar floating point type gets softened to integer and
482 // then promoted to a larger integer. If PartEVT is the larger integer
483 // we need to truncate it and then bitcast to the FP type.
484 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
485 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
486 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
487 Val = DAG.getBitcast(ValueSVT, Val);
488 } else {
489 Val = ValueVT.isFloatingPoint()
490 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
491 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
492 }
493 }
494
495 return DAG.getBuildVector(ValueVT, DL, Val);
496}
497
498static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
499 SDValue Val, SDValue *Parts, unsigned NumParts,
500 MVT PartVT, const Value *V,
501 std::optional<CallingConv::ID> CallConv);
502
503/// getCopyToParts - Create a series of nodes that contain the specified value
504/// split into legal parts. If the parts contain more bits than Val, then, for
505/// integers, ExtendKind can be used to specify how to generate the extra bits.
506static void
508 unsigned NumParts, MVT PartVT, const Value *V,
509 std::optional<CallingConv::ID> CallConv = std::nullopt,
510 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
511 // Let the target split the parts if it wants to
512 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
513 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
514 CallConv))
515 return;
516 EVT ValueVT = Val.getValueType();
517
518 // Handle the vector case separately.
519 if (ValueVT.isVector())
520 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
521 CallConv);
522
523 unsigned OrigNumParts = NumParts;
525 "Copying to an illegal type!");
526
527 if (NumParts == 0)
528 return;
529
530 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
531 EVT PartEVT = PartVT;
532 if (PartEVT == ValueVT) {
533 assert(NumParts == 1 && "No-op copy with multiple parts!");
534 Parts[0] = Val;
535 return;
536 }
537
538 unsigned PartBits = PartVT.getSizeInBits();
539 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
540 // If the parts cover more bits than the value has, promote the value.
541 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
542 assert(NumParts == 1 && "Do not know what to promote to!");
543 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
544 } else {
545 if (ValueVT.isFloatingPoint()) {
546 // FP values need to be bitcast, then extended if they are being put
547 // into a larger container.
548 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
549 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
550 }
551 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
552 ValueVT.isInteger() &&
553 "Unknown mismatch!");
554 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
555 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
556 if (PartVT == MVT::x86mmx)
557 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
558 }
559 } else if (PartBits == ValueVT.getSizeInBits()) {
560 // Different types of the same size.
561 assert(NumParts == 1 && PartEVT != ValueVT);
562 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
563 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
564 // If the parts cover less bits than value has, truncate the value.
565 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
566 ValueVT.isInteger() &&
567 "Unknown mismatch!");
568 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
569 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
570 if (PartVT == MVT::x86mmx)
571 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
572 }
573
574 // The value may have changed - recompute ValueVT.
575 ValueVT = Val.getValueType();
576 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
577 "Failed to tile the value with PartVT!");
578
579 if (NumParts == 1) {
580 if (PartEVT != ValueVT) {
582 "scalar-to-vector conversion failed");
583 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
584 }
585
586 Parts[0] = Val;
587 return;
588 }
589
590 // Expand the value into multiple parts.
591 if (NumParts & (NumParts - 1)) {
592 // The number of parts is not a power of 2. Split off and copy the tail.
593 assert(PartVT.isInteger() && ValueVT.isInteger() &&
594 "Do not know what to expand to!");
595 unsigned RoundParts = llvm::bit_floor(NumParts);
596 unsigned RoundBits = RoundParts * PartBits;
597 unsigned OddParts = NumParts - RoundParts;
598 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
599 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
600
601 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
602 CallConv);
603
604 if (DAG.getDataLayout().isBigEndian())
605 // The odd parts were reversed by getCopyToParts - unreverse them.
606 std::reverse(Parts + RoundParts, Parts + NumParts);
607
608 NumParts = RoundParts;
609 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
610 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
611 }
612
613 // The number of parts is a power of 2. Repeatedly bisect the value using
614 // EXTRACT_ELEMENT.
615 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
617 ValueVT.getSizeInBits()),
618 Val);
619
620 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
621 for (unsigned i = 0; i < NumParts; i += StepSize) {
622 unsigned ThisBits = StepSize * PartBits / 2;
623 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
624 SDValue &Part0 = Parts[i];
625 SDValue &Part1 = Parts[i+StepSize/2];
626
627 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
628 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
629 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
630 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
631
632 if (ThisBits == PartBits && ThisVT != PartVT) {
633 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
634 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
635 }
636 }
637 }
638
639 if (DAG.getDataLayout().isBigEndian())
640 std::reverse(Parts, Parts + OrigNumParts);
641}
642
644 const SDLoc &DL, EVT PartVT) {
645 if (!PartVT.isVector())
646 return SDValue();
647
648 EVT ValueVT = Val.getValueType();
649 EVT PartEVT = PartVT.getVectorElementType();
650 EVT ValueEVT = ValueVT.getVectorElementType();
651 ElementCount PartNumElts = PartVT.getVectorElementCount();
652 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
653
654 // We only support widening vectors with equivalent element types and
655 // fixed/scalable properties. If a target needs to widen a fixed-length type
656 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
657 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
658 PartNumElts.isScalable() != ValueNumElts.isScalable())
659 return SDValue();
660
661 // Have a try for bf16 because some targets share its ABI with fp16.
662 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
664 "Cannot widen to illegal type");
665 Val = DAG.getNode(ISD::BITCAST, DL,
666 ValueVT.changeVectorElementType(MVT::f16), Val);
667 } else if (PartEVT != ValueEVT) {
668 return SDValue();
669 }
670
671 // Widening a scalable vector to another scalable vector is done by inserting
672 // the vector into a larger undef one.
673 if (PartNumElts.isScalable())
674 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
675 Val, DAG.getVectorIdxConstant(0, DL));
676
677 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
678 // undef elements.
680 DAG.ExtractVectorElements(Val, Ops);
681 SDValue EltUndef = DAG.getUNDEF(PartEVT);
682 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
683
684 // FIXME: Use CONCAT for 2x -> 4x.
685 return DAG.getBuildVector(PartVT, DL, Ops);
686}
687
688/// getCopyToPartsVector - Create a series of nodes that contain the specified
689/// value split into legal parts.
690static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
691 SDValue Val, SDValue *Parts, unsigned NumParts,
692 MVT PartVT, const Value *V,
693 std::optional<CallingConv::ID> CallConv) {
694 EVT ValueVT = Val.getValueType();
695 assert(ValueVT.isVector() && "Not a vector");
696 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
697 const bool IsABIRegCopy = CallConv.has_value();
698
699 if (NumParts == 1) {
700 EVT PartEVT = PartVT;
701 if (PartEVT == ValueVT) {
702 // Nothing to do.
703 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
704 // Bitconvert vector->vector case.
705 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
706 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
707 Val = Widened;
708 } else if (PartVT.isVector() &&
710 ValueVT.getVectorElementType()) &&
711 PartEVT.getVectorElementCount() ==
712 ValueVT.getVectorElementCount()) {
713
714 // Promoted vector extract
715 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
716 } else if (PartEVT.isVector() &&
717 PartEVT.getVectorElementType() !=
718 ValueVT.getVectorElementType() &&
719 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
720 TargetLowering::TypeWidenVector) {
721 // Combination of widening and promotion.
722 EVT WidenVT =
724 PartVT.getVectorElementCount());
725 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
726 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
727 } else {
728 // Don't extract an integer from a float vector. This can happen if the
729 // FP type gets softened to integer and then promoted. The promotion
730 // prevents it from being picked up by the earlier bitcast case.
731 if (ValueVT.getVectorElementCount().isScalar() &&
732 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
733 // If we reach this condition and PartVT is FP, this means that
734 // ValueVT is also FP and both have a different size, otherwise we
735 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
736 // would be invalid since that would mean the smaller FP type has to
737 // be extended to the larger one.
738 if (PartVT.isFloatingPoint()) {
739 Val = DAG.getBitcast(ValueVT.getScalarType(), Val);
740 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
741 } else
742 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
743 DAG.getVectorIdxConstant(0, DL));
744 } else {
745 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
746 assert(PartVT.getFixedSizeInBits() > ValueSize &&
747 "lossy conversion of vector to scalar type");
748 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
749 Val = DAG.getBitcast(IntermediateType, Val);
750 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
751 }
752 }
753
754 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
755 Parts[0] = Val;
756 return;
757 }
758
759 // Handle a multi-element vector.
760 EVT IntermediateVT;
761 MVT RegisterVT;
762 unsigned NumIntermediates;
763 unsigned NumRegs;
764 if (IsABIRegCopy) {
766 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
767 RegisterVT);
768 } else {
769 NumRegs =
770 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
771 NumIntermediates, RegisterVT);
772 }
773
774 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
775 NumParts = NumRegs; // Silence a compiler warning.
776 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
777
778 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
779 "Mixing scalable and fixed vectors when copying in parts");
780
781 std::optional<ElementCount> DestEltCnt;
782
783 if (IntermediateVT.isVector())
784 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
785 else
786 DestEltCnt = ElementCount::getFixed(NumIntermediates);
787
788 EVT BuiltVectorTy = EVT::getVectorVT(
789 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
790
791 if (ValueVT == BuiltVectorTy) {
792 // Nothing to do.
793 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
794 // Bitconvert vector->vector case.
795 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
796 } else {
797 if (BuiltVectorTy.getVectorElementType().bitsGT(
798 ValueVT.getVectorElementType())) {
799 // Integer promotion.
800 ValueVT = EVT::getVectorVT(*DAG.getContext(),
801 BuiltVectorTy.getVectorElementType(),
802 ValueVT.getVectorElementCount());
803 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
804 }
805
806 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
807 Val = Widened;
808 }
809 }
810
811 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
812
813 // Split the vector into intermediate operands.
814 SmallVector<SDValue, 8> Ops(NumIntermediates);
815 for (unsigned i = 0; i != NumIntermediates; ++i) {
816 if (IntermediateVT.isVector()) {
817 // This does something sensible for scalable vectors - see the
818 // definition of EXTRACT_SUBVECTOR for further details.
819 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
820 Ops[i] =
821 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
822 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
823 } else {
824 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
825 DAG.getVectorIdxConstant(i, DL));
826 }
827 }
828
829 // Split the intermediate operands into legal parts.
830 if (NumParts == NumIntermediates) {
831 // If the register was not expanded, promote or copy the value,
832 // as appropriate.
833 for (unsigned i = 0; i != NumParts; ++i)
834 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
835 } else if (NumParts > 0) {
836 // If the intermediate type was expanded, split each the value into
837 // legal parts.
838 assert(NumIntermediates != 0 && "division by zero");
839 assert(NumParts % NumIntermediates == 0 &&
840 "Must expand into a divisible number of parts!");
841 unsigned Factor = NumParts / NumIntermediates;
842 for (unsigned i = 0; i != NumIntermediates; ++i)
843 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
844 CallConv);
845 }
846}
847
849 EVT valuevt, std::optional<CallingConv::ID> CC)
850 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
851 RegCount(1, regs.size()), CallConv(CC) {}
852
854 const DataLayout &DL, unsigned Reg, Type *Ty,
855 std::optional<CallingConv::ID> CC) {
856 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
857
858 CallConv = CC;
859
860 for (EVT ValueVT : ValueVTs) {
861 unsigned NumRegs =
863 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
864 : TLI.getNumRegisters(Context, ValueVT);
865 MVT RegisterVT =
867 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
868 : TLI.getRegisterType(Context, ValueVT);
869 for (unsigned i = 0; i != NumRegs; ++i)
870 Regs.push_back(Reg + i);
871 RegVTs.push_back(RegisterVT);
872 RegCount.push_back(NumRegs);
873 Reg += NumRegs;
874 }
875}
876
878 FunctionLoweringInfo &FuncInfo,
879 const SDLoc &dl, SDValue &Chain,
880 SDValue *Glue, const Value *V) const {
881 // A Value with type {} or [0 x %t] needs no registers.
882 if (ValueVTs.empty())
883 return SDValue();
884
885 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
886
887 // Assemble the legal parts into the final values.
888 SmallVector<SDValue, 4> Values(ValueVTs.size());
890 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
891 // Copy the legal parts from the registers.
892 EVT ValueVT = ValueVTs[Value];
893 unsigned NumRegs = RegCount[Value];
894 MVT RegisterVT = isABIMangled()
896 *DAG.getContext(), *CallConv, RegVTs[Value])
897 : RegVTs[Value];
898
899 Parts.resize(NumRegs);
900 for (unsigned i = 0; i != NumRegs; ++i) {
901 SDValue P;
902 if (!Glue) {
903 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
904 } else {
905 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
906 *Glue = P.getValue(2);
907 }
908
909 Chain = P.getValue(1);
910 Parts[i] = P;
911
912 // If the source register was virtual and if we know something about it,
913 // add an assert node.
914 if (!Register::isVirtualRegister(Regs[Part + i]) ||
915 !RegisterVT.isInteger())
916 continue;
917
919 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
920 if (!LOI)
921 continue;
922
923 unsigned RegSize = RegisterVT.getScalarSizeInBits();
924 unsigned NumSignBits = LOI->NumSignBits;
925 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
926
927 if (NumZeroBits == RegSize) {
928 // The current value is a zero.
929 // Explicitly express that as it would be easier for
930 // optimizations to kick in.
931 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
932 continue;
933 }
934
935 // FIXME: We capture more information than the dag can represent. For
936 // now, just use the tightest assertzext/assertsext possible.
937 bool isSExt;
938 EVT FromVT(MVT::Other);
939 if (NumZeroBits) {
940 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
941 isSExt = false;
942 } else if (NumSignBits > 1) {
943 FromVT =
944 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
945 isSExt = true;
946 } else {
947 continue;
948 }
949 // Add an assertion node.
950 assert(FromVT != MVT::Other);
951 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
952 RegisterVT, P, DAG.getValueType(FromVT));
953 }
954
955 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
956 RegisterVT, ValueVT, V, Chain, CallConv);
957 Part += NumRegs;
958 Parts.clear();
959 }
960
961 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
962}
963
965 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
966 const Value *V,
967 ISD::NodeType PreferredExtendType) const {
968 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
969 ISD::NodeType ExtendKind = PreferredExtendType;
970
971 // Get the list of the values's legal parts.
972 unsigned NumRegs = Regs.size();
973 SmallVector<SDValue, 8> Parts(NumRegs);
974 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
975 unsigned NumParts = RegCount[Value];
976
977 MVT RegisterVT = isABIMangled()
979 *DAG.getContext(), *CallConv, RegVTs[Value])
980 : RegVTs[Value];
981
982 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
983 ExtendKind = ISD::ZERO_EXTEND;
984
985 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
986 NumParts, RegisterVT, V, CallConv, ExtendKind);
987 Part += NumParts;
988 }
989
990 // Copy the parts into the registers.
991 SmallVector<SDValue, 8> Chains(NumRegs);
992 for (unsigned i = 0; i != NumRegs; ++i) {
993 SDValue Part;
994 if (!Glue) {
995 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
996 } else {
997 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
998 *Glue = Part.getValue(1);
999 }
1000
1001 Chains[i] = Part.getValue(0);
1002 }
1003
1004 if (NumRegs == 1 || Glue)
1005 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1006 // flagged to it. That is the CopyToReg nodes and the user are considered
1007 // a single scheduling unit. If we create a TokenFactor and return it as
1008 // chain, then the TokenFactor is both a predecessor (operand) of the
1009 // user as well as a successor (the TF operands are flagged to the user).
1010 // c1, f1 = CopyToReg
1011 // c2, f2 = CopyToReg
1012 // c3 = TokenFactor c1, c2
1013 // ...
1014 // = op c3, ..., f2
1015 Chain = Chains[NumRegs-1];
1016 else
1017 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1018}
1019
1021 unsigned MatchingIdx, const SDLoc &dl,
1022 SelectionDAG &DAG,
1023 std::vector<SDValue> &Ops) const {
1024 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1025
1026 InlineAsm::Flag Flag(Code, Regs.size());
1027 if (HasMatching)
1028 Flag.setMatchingOp(MatchingIdx);
1029 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
1030 // Put the register class of the virtual registers in the flag word. That
1031 // way, later passes can recompute register class constraints for inline
1032 // assembly as well as normal instructions.
1033 // Don't do this for tied operands that can use the regclass information
1034 // from the def.
1036 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1037 Flag.setRegClass(RC->getID());
1038 }
1039
1040 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1041 Ops.push_back(Res);
1042
1043 if (Code == InlineAsm::Kind::Clobber) {
1044 // Clobbers should always have a 1:1 mapping with registers, and may
1045 // reference registers that have illegal (e.g. vector) types. Hence, we
1046 // shouldn't try to apply any sort of splitting logic to them.
1047 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1048 "No 1:1 mapping from clobbers to regs?");
1050 (void)SP;
1051 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1052 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1053 assert(
1054 (Regs[I] != SP ||
1056 "If we clobbered the stack pointer, MFI should know about it.");
1057 }
1058 return;
1059 }
1060
1061 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1062 MVT RegisterVT = RegVTs[Value];
1063 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1064 RegisterVT);
1065 for (unsigned i = 0; i != NumRegs; ++i) {
1066 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1067 unsigned TheReg = Regs[Reg++];
1068 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1069 }
1070 }
1071}
1072
1076 unsigned I = 0;
1077 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1078 unsigned RegCount = std::get<0>(CountAndVT);
1079 MVT RegisterVT = std::get<1>(CountAndVT);
1080 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1081 for (unsigned E = I + RegCount; I != E; ++I)
1082 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1083 }
1084 return OutVec;
1085}
1086
1088 AssumptionCache *ac,
1089 const TargetLibraryInfo *li) {
1090 AA = aa;
1091 AC = ac;
1092 GFI = gfi;
1093 LibInfo = li;
1095 LPadToCallSiteMap.clear();
1097 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1099}
1100
1102 NodeMap.clear();
1103 UnusedArgNodeMap.clear();
1104 PendingLoads.clear();
1105 PendingExports.clear();
1106 PendingConstrainedFP.clear();
1107 PendingConstrainedFPStrict.clear();
1108 CurInst = nullptr;
1109 HasTailCall = false;
1110 SDNodeOrder = LowestSDNodeOrder;
1112}
1113
1115 DanglingDebugInfoMap.clear();
1116}
1117
1118// Update DAG root to include dependencies on Pending chains.
1119SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1120 SDValue Root = DAG.getRoot();
1121
1122 if (Pending.empty())
1123 return Root;
1124
1125 // Add current root to PendingChains, unless we already indirectly
1126 // depend on it.
1127 if (Root.getOpcode() != ISD::EntryToken) {
1128 unsigned i = 0, e = Pending.size();
1129 for (; i != e; ++i) {
1130 assert(Pending[i].getNode()->getNumOperands() > 1);
1131 if (Pending[i].getNode()->getOperand(0) == Root)
1132 break; // Don't add the root if we already indirectly depend on it.
1133 }
1134
1135 if (i == e)
1136 Pending.push_back(Root);
1137 }
1138
1139 if (Pending.size() == 1)
1140 Root = Pending[0];
1141 else
1142 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1143
1144 DAG.setRoot(Root);
1145 Pending.clear();
1146 return Root;
1147}
1148
1150 return updateRoot(PendingLoads);
1151}
1152
1154 // Chain up all pending constrained intrinsics together with all
1155 // pending loads, by simply appending them to PendingLoads and
1156 // then calling getMemoryRoot().
1157 PendingLoads.reserve(PendingLoads.size() +
1158 PendingConstrainedFP.size() +
1159 PendingConstrainedFPStrict.size());
1160 PendingLoads.append(PendingConstrainedFP.begin(),
1161 PendingConstrainedFP.end());
1162 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1163 PendingConstrainedFPStrict.end());
1164 PendingConstrainedFP.clear();
1165 PendingConstrainedFPStrict.clear();
1166 return getMemoryRoot();
1167}
1168
1170 // We need to emit pending fpexcept.strict constrained intrinsics,
1171 // so append them to the PendingExports list.
1172 PendingExports.append(PendingConstrainedFPStrict.begin(),
1173 PendingConstrainedFPStrict.end());
1174 PendingConstrainedFPStrict.clear();
1175 return updateRoot(PendingExports);
1176}
1177
1179 DILocalVariable *Variable,
1181 DebugLoc DL) {
1182 assert(Variable && "Missing variable");
1183
1184 // Check if address has undef value.
1185 if (!Address || isa<UndefValue>(Address) ||
1186 (Address->use_empty() && !isa<Argument>(Address))) {
1187 LLVM_DEBUG(
1188 dbgs()
1189 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1190 return;
1191 }
1192
1193 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1194
1195 SDValue &N = NodeMap[Address];
1196 if (!N.getNode() && isa<Argument>(Address))
1197 // Check unused arguments map.
1198 N = UnusedArgNodeMap[Address];
1199 SDDbgValue *SDV;
1200 if (N.getNode()) {
1201 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1202 Address = BCI->getOperand(0);
1203 // Parameters are handled specially.
1204 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1205 if (IsParameter && FINode) {
1206 // Byval parameter. We have a frame index at this point.
1207 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1208 /*IsIndirect*/ true, DL, SDNodeOrder);
1209 } else if (isa<Argument>(Address)) {
1210 // Address is an argument, so try to emit its dbg value using
1211 // virtual register info from the FuncInfo.ValueMap.
1212 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1213 FuncArgumentDbgValueKind::Declare, N);
1214 return;
1215 } else {
1216 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1217 true, DL, SDNodeOrder);
1218 }
1219 DAG.AddDbgValue(SDV, IsParameter);
1220 } else {
1221 // If Address is an argument then try to emit its dbg value using
1222 // virtual register info from the FuncInfo.ValueMap.
1223 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1224 FuncArgumentDbgValueKind::Declare, N)) {
1225 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1226 << " (could not emit func-arg dbg_value)\n");
1227 }
1228 }
1229 return;
1230}
1231
1233 // Add SDDbgValue nodes for any var locs here. Do so before updating
1234 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1235 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1236 // Add SDDbgValue nodes for any var locs here. Do so before updating
1237 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1238 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1239 It != End; ++It) {
1240 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1241 dropDanglingDebugInfo(Var, It->Expr);
1242 if (It->Values.isKillLocation(It->Expr)) {
1243 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1244 continue;
1245 }
1246 SmallVector<Value *> Values(It->Values.location_ops());
1247 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1248 It->Values.hasArgList())) {
1249 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1251 FnVarLocs->getDILocalVariable(It->VariableID),
1252 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1253 }
1254 }
1255 }
1256
1257 // We must skip DbgVariableRecords if they've already been processed above as
1258 // we have just emitted the debug values resulting from assignment tracking
1259 // analysis, making any existing DbgVariableRecords redundant (and probably
1260 // less correct). We still need to process DbgLabelRecords. This does sink
1261 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1262 // be important as it does so deterministcally and ordering between
1263 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1264 // printing).
1265 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1266 // Is there is any debug-info attached to this instruction, in the form of
1267 // DbgRecord non-instruction debug-info records.
1268 for (DbgRecord &DR : I.getDbgRecordRange()) {
1269 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1270 assert(DLR->getLabel() && "Missing label");
1271 SDDbgLabel *SDV =
1272 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1273 DAG.AddDbgLabel(SDV);
1274 continue;
1275 }
1276
1277 if (SkipDbgVariableRecords)
1278 continue;
1279 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
1280 DILocalVariable *Variable = DVR.getVariable();
1283
1285 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1286 continue;
1287 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1288 << "\n");
1290 DVR.getDebugLoc());
1291 continue;
1292 }
1293
1294 // A DbgVariableRecord with no locations is a kill location.
1296 if (Values.empty()) {
1298 SDNodeOrder);
1299 continue;
1300 }
1301
1302 // A DbgVariableRecord with an undef or absent location is also a kill
1303 // location.
1304 if (llvm::any_of(Values,
1305 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1307 SDNodeOrder);
1308 continue;
1309 }
1310
1311 bool IsVariadic = DVR.hasArgList();
1312 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1313 SDNodeOrder, IsVariadic)) {
1314 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1315 DVR.getDebugLoc(), SDNodeOrder);
1316 }
1317 }
1318}
1319
1321 visitDbgInfo(I);
1322
1323 // Set up outgoing PHI node register values before emitting the terminator.
1324 if (I.isTerminator()) {
1325 HandlePHINodesInSuccessorBlocks(I.getParent());
1326 }
1327
1328 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1329 if (!isa<DbgInfoIntrinsic>(I))
1330 ++SDNodeOrder;
1331
1332 CurInst = &I;
1333
1334 // Set inserted listener only if required.
1335 bool NodeInserted = false;
1336 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1337 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1338 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);
1339 if (PCSectionsMD || MMRA) {
1340 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1341 DAG, [&](SDNode *) { NodeInserted = true; });
1342 }
1343
1344 visit(I.getOpcode(), I);
1345
1346 if (!I.isTerminator() && !HasTailCall &&
1347 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1349
1350 // Handle metadata.
1351 if (PCSectionsMD || MMRA) {
1352 auto It = NodeMap.find(&I);
1353 if (It != NodeMap.end()) {
1354 if (PCSectionsMD)
1355 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1356 if (MMRA)
1357 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1358 } else if (NodeInserted) {
1359 // This should not happen; if it does, don't let it go unnoticed so we can
1360 // fix it. Relevant visit*() function is probably missing a setValue().
1361 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1362 << I.getModule()->getName() << "]\n";
1363 LLVM_DEBUG(I.dump());
1364 assert(false);
1365 }
1366 }
1367
1368 CurInst = nullptr;
1369}
1370
1371void SelectionDAGBuilder::visitPHI(const PHINode &) {
1372 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1373}
1374
1375void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1376 // Note: this doesn't use InstVisitor, because it has to work with
1377 // ConstantExpr's in addition to instructions.
1378 switch (Opcode) {
1379 default: llvm_unreachable("Unknown instruction type encountered!");
1380 // Build the switch statement using the Instruction.def file.
1381#define HANDLE_INST(NUM, OPCODE, CLASS) \
1382 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1383#include "llvm/IR/Instruction.def"
1384 }
1385}
1386
1388 DILocalVariable *Variable,
1389 DebugLoc DL, unsigned Order,
1392 // For variadic dbg_values we will now insert an undef.
1393 // FIXME: We can potentially recover these!
1395 for (const Value *V : Values) {
1396 auto *Undef = UndefValue::get(V->getType());
1398 }
1399 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1400 /*IsIndirect=*/false, DL, Order,
1401 /*IsVariadic=*/true);
1402 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1403 return true;
1404}
1405
1407 DILocalVariable *Var,
1408 DIExpression *Expr,
1409 bool IsVariadic, DebugLoc DL,
1410 unsigned Order) {
1411 if (IsVariadic) {
1412 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1413 return;
1414 }
1415 // TODO: Dangling debug info will eventually either be resolved or produce
1416 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1417 // between the original dbg.value location and its resolved DBG_VALUE,
1418 // which we should ideally fill with an extra Undef DBG_VALUE.
1419 assert(Values.size() == 1);
1420 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1421}
1422
1424 const DIExpression *Expr) {
1425 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1426 DIVariable *DanglingVariable = DDI.getVariable();
1427 DIExpression *DanglingExpr = DDI.getExpression();
1428 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1429 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1430 << printDDI(nullptr, DDI) << "\n");
1431 return true;
1432 }
1433 return false;
1434 };
1435
1436 for (auto &DDIMI : DanglingDebugInfoMap) {
1437 DanglingDebugInfoVector &DDIV = DDIMI.second;
1438
1439 // If debug info is to be dropped, run it through final checks to see
1440 // whether it can be salvaged.
1441 for (auto &DDI : DDIV)
1442 if (isMatchingDbgValue(DDI))
1443 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1444
1445 erase_if(DDIV, isMatchingDbgValue);
1446 }
1447}
1448
1449// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1450// generate the debug data structures now that we've seen its definition.
1452 SDValue Val) {
1453 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1454 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1455 return;
1456
1457 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1458 for (auto &DDI : DDIV) {
1459 DebugLoc DL = DDI.getDebugLoc();
1460 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1461 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1462 DILocalVariable *Variable = DDI.getVariable();
1463 DIExpression *Expr = DDI.getExpression();
1465 "Expected inlined-at fields to agree");
1466 SDDbgValue *SDV;
1467 if (Val.getNode()) {
1468 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1469 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1470 // we couldn't resolve it directly when examining the DbgValue intrinsic
1471 // in the first place we should not be more successful here). Unless we
1472 // have some test case that prove this to be correct we should avoid
1473 // calling EmitFuncArgumentDbgValue here.
1474 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1475 FuncArgumentDbgValueKind::Value, Val)) {
1476 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1477 << printDDI(V, DDI) << "\n");
1478 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1479 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1480 // inserted after the definition of Val when emitting the instructions
1481 // after ISel. An alternative could be to teach
1482 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1483 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1484 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1485 << ValSDNodeOrder << "\n");
1486 SDV = getDbgValue(Val, Variable, Expr, DL,
1487 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1488 DAG.AddDbgValue(SDV, false);
1489 } else
1490 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1491 << printDDI(V, DDI)
1492 << " in EmitFuncArgumentDbgValue\n");
1493 } else {
1494 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1495 << "\n");
1496 auto Undef = UndefValue::get(V->getType());
1497 auto SDV =
1498 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1499 DAG.AddDbgValue(SDV, false);
1500 }
1501 }
1502 DDIV.clear();
1503}
1504
1506 DanglingDebugInfo &DDI) {
1507 // TODO: For the variadic implementation, instead of only checking the fail
1508 // state of `handleDebugValue`, we need know specifically which values were
1509 // invalid, so that we attempt to salvage only those values when processing
1510 // a DIArgList.
1511 const Value *OrigV = V;
1512 DILocalVariable *Var = DDI.getVariable();
1513 DIExpression *Expr = DDI.getExpression();
1514 DebugLoc DL = DDI.getDebugLoc();
1515 unsigned SDOrder = DDI.getSDNodeOrder();
1516
1517 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1518 // that DW_OP_stack_value is desired.
1519 bool StackValue = true;
1520
1521 // Can this Value can be encoded without any further work?
1522 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1523 return;
1524
1525 // Attempt to salvage back through as many instructions as possible. Bail if
1526 // a non-instruction is seen, such as a constant expression or global
1527 // variable. FIXME: Further work could recover those too.
1528 while (isa<Instruction>(V)) {
1529 const Instruction &VAsInst = *cast<const Instruction>(V);
1530 // Temporary "0", awaiting real implementation.
1532 SmallVector<Value *, 4> AdditionalValues;
1533 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1534 Expr->getNumLocationOperands(), Ops,
1535 AdditionalValues);
1536 // If we cannot salvage any further, and haven't yet found a suitable debug
1537 // expression, bail out.
1538 if (!V)
1539 break;
1540
1541 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1542 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1543 // here for variadic dbg_values, remove that condition.
1544 if (!AdditionalValues.empty())
1545 break;
1546
1547 // New value and expr now represent this debuginfo.
1548 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1549
1550 // Some kind of simplification occurred: check whether the operand of the
1551 // salvaged debug expression can be encoded in this DAG.
1552 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1553 LLVM_DEBUG(
1554 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1555 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1556 return;
1557 }
1558 }
1559
1560 // This was the final opportunity to salvage this debug information, and it
1561 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1562 // any earlier variable location.
1563 assert(OrigV && "V shouldn't be null");
1564 auto *Undef = UndefValue::get(OrigV->getType());
1565 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1566 DAG.AddDbgValue(SDV, false);
1567 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1568 << printDDI(OrigV, DDI) << "\n");
1569}
1570
1572 DIExpression *Expr,
1573 DebugLoc DbgLoc,
1574 unsigned Order) {
1578 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1579 /*IsVariadic*/ false);
1580}
1581
1583 DILocalVariable *Var,
1584 DIExpression *Expr, DebugLoc DbgLoc,
1585 unsigned Order, bool IsVariadic) {
1586 if (Values.empty())
1587 return true;
1588
1589 // Filter EntryValue locations out early.
1590 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1591 return true;
1592
1593 SmallVector<SDDbgOperand> LocationOps;
1594 SmallVector<SDNode *> Dependencies;
1595 for (const Value *V : Values) {
1596 // Constant value.
1597 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1598 isa<ConstantPointerNull>(V)) {
1599 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1600 continue;
1601 }
1602
1603 // Look through IntToPtr constants.
1604 if (auto *CE = dyn_cast<ConstantExpr>(V))
1605 if (CE->getOpcode() == Instruction::IntToPtr) {
1606 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1607 continue;
1608 }
1609
1610 // If the Value is a frame index, we can create a FrameIndex debug value
1611 // without relying on the DAG at all.
1612 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1613 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1614 if (SI != FuncInfo.StaticAllocaMap.end()) {
1615 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1616 continue;
1617 }
1618 }
1619
1620 // Do not use getValue() in here; we don't want to generate code at
1621 // this point if it hasn't been done yet.
1622 SDValue N = NodeMap[V];
1623 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1624 N = UnusedArgNodeMap[V];
1625 if (N.getNode()) {
1626 // Only emit func arg dbg value for non-variadic dbg.values for now.
1627 if (!IsVariadic &&
1628 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1629 FuncArgumentDbgValueKind::Value, N))
1630 return true;
1631 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1632 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1633 // describe stack slot locations.
1634 //
1635 // Consider "int x = 0; int *px = &x;". There are two kinds of
1636 // interesting debug values here after optimization:
1637 //
1638 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1639 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1640 //
1641 // Both describe the direct values of their associated variables.
1642 Dependencies.push_back(N.getNode());
1643 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1644 continue;
1645 }
1646 LocationOps.emplace_back(
1647 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1648 continue;
1649 }
1650
1652 // Special rules apply for the first dbg.values of parameter variables in a
1653 // function. Identify them by the fact they reference Argument Values, that
1654 // they're parameters, and they are parameters of the current function. We
1655 // need to let them dangle until they get an SDNode.
1656 bool IsParamOfFunc =
1657 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1658 if (IsParamOfFunc)
1659 return false;
1660
1661 // The value is not used in this block yet (or it would have an SDNode).
1662 // We still want the value to appear for the user if possible -- if it has
1663 // an associated VReg, we can refer to that instead.
1664 auto VMI = FuncInfo.ValueMap.find(V);
1665 if (VMI != FuncInfo.ValueMap.end()) {
1666 unsigned Reg = VMI->second;
1667 // If this is a PHI node, it may be split up into several MI PHI nodes
1668 // (in FunctionLoweringInfo::set).
1669 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1670 V->getType(), std::nullopt);
1671 if (RFV.occupiesMultipleRegs()) {
1672 // FIXME: We could potentially support variadic dbg_values here.
1673 if (IsVariadic)
1674 return false;
1675 unsigned Offset = 0;
1676 unsigned BitsToDescribe = 0;
1677 if (auto VarSize = Var->getSizeInBits())
1678 BitsToDescribe = *VarSize;
1679 if (auto Fragment = Expr->getFragmentInfo())
1680 BitsToDescribe = Fragment->SizeInBits;
1681 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1682 // Bail out if all bits are described already.
1683 if (Offset >= BitsToDescribe)
1684 break;
1685 // TODO: handle scalable vectors.
1686 unsigned RegisterSize = RegAndSize.second;
1687 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1688 ? BitsToDescribe - Offset
1689 : RegisterSize;
1690 auto FragmentExpr = DIExpression::createFragmentExpression(
1691 Expr, Offset, FragmentSize);
1692 if (!FragmentExpr)
1693 continue;
1695 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);
1696 DAG.AddDbgValue(SDV, false);
1697 Offset += RegisterSize;
1698 }
1699 return true;
1700 }
1701 // We can use simple vreg locations for variadic dbg_values as well.
1702 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1703 continue;
1704 }
1705 // We failed to create a SDDbgOperand for V.
1706 return false;
1707 }
1708
1709 // We have created a SDDbgOperand for each Value in Values.
1710 assert(!LocationOps.empty());
1711 SDDbgValue *SDV =
1712 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1713 /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
1714 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1715 return true;
1716}
1717
1719 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1720 for (auto &Pair : DanglingDebugInfoMap)
1721 for (auto &DDI : Pair.second)
1722 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1724}
1725
1726/// getCopyFromRegs - If there was virtual register allocated for the value V
1727/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1730 SDValue Result;
1731
1732 if (It != FuncInfo.ValueMap.end()) {
1733 Register InReg = It->second;
1734
1736 DAG.getDataLayout(), InReg, Ty,
1737 std::nullopt); // This is not an ABI copy.
1738 SDValue Chain = DAG.getEntryNode();
1739 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1740 V);
1741 resolveDanglingDebugInfo(V, Result);
1742 }
1743
1744 return Result;
1745}
1746
1747/// getValue - Return an SDValue for the given Value.
1749 // If we already have an SDValue for this value, use it. It's important
1750 // to do this first, so that we don't create a CopyFromReg if we already
1751 // have a regular SDValue.
1752 SDValue &N = NodeMap[V];
1753 if (N.getNode()) return N;
1754
1755 // If there's a virtual register allocated and initialized for this
1756 // value, use it.
1757 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1758 return copyFromReg;
1759
1760 // Otherwise create a new SDValue and remember it.
1761 SDValue Val = getValueImpl(V);
1762 NodeMap[V] = Val;
1764 return Val;
1765}
1766
1767/// getNonRegisterValue - Return an SDValue for the given Value, but
1768/// don't look in FuncInfo.ValueMap for a virtual register.
1770 // If we already have an SDValue for this value, use it.
1771 SDValue &N = NodeMap[V];
1772 if (N.getNode()) {
1773 if (isIntOrFPConstant(N)) {
1774 // Remove the debug location from the node as the node is about to be used
1775 // in a location which may differ from the original debug location. This
1776 // is relevant to Constant and ConstantFP nodes because they can appear
1777 // as constant expressions inside PHI nodes.
1778 N->setDebugLoc(DebugLoc());
1779 }
1780 return N;
1781 }
1782
1783 // Otherwise create a new SDValue and remember it.
1784 SDValue Val = getValueImpl(V);
1785 NodeMap[V] = Val;
1787 return Val;
1788}
1789
1790/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1791/// Create an SDValue for the given value.
1794
1795 if (const Constant *C = dyn_cast<Constant>(V)) {
1796 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1797
1798 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1799 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1800
1801 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1802 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1803
1804 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
1806 getValue(CPA->getPointer()), getValue(CPA->getKey()),
1807 getValue(CPA->getAddrDiscriminator()),
1808 getValue(CPA->getDiscriminator()));
1809 }
1810
1811 if (isa<ConstantPointerNull>(C)) {
1812 unsigned AS = V->getType()->getPointerAddressSpace();
1813 return DAG.getConstant(0, getCurSDLoc(),
1814 TLI.getPointerTy(DAG.getDataLayout(), AS));
1815 }
1816
1817 if (match(C, m_VScale()))
1818 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1819
1820 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1821 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1822
1823 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1824 return DAG.getUNDEF(VT);
1825
1826 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1827 visit(CE->getOpcode(), *CE);
1828 SDValue N1 = NodeMap[V];
1829 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1830 return N1;
1831 }
1832
1833 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1835 for (const Use &U : C->operands()) {
1836 SDNode *Val = getValue(U).getNode();
1837 // If the operand is an empty aggregate, there are no values.
1838 if (!Val) continue;
1839 // Add each leaf value from the operand to the Constants list
1840 // to form a flattened list of all the values.
1841 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1842 Constants.push_back(SDValue(Val, i));
1843 }
1844
1846 }
1847
1848 if (const ConstantDataSequential *CDS =
1849 dyn_cast<ConstantDataSequential>(C)) {
1851 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1852 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1853 // Add each leaf value from the operand to the Constants list
1854 // to form a flattened list of all the values.
1855 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1856 Ops.push_back(SDValue(Val, i));
1857 }
1858
1859 if (isa<ArrayType>(CDS->getType()))
1860 return DAG.getMergeValues(Ops, getCurSDLoc());
1861 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1862 }
1863
1864 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1865 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1866 "Unknown struct or array constant!");
1867
1868 SmallVector<EVT, 4> ValueVTs;
1869 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1870 unsigned NumElts = ValueVTs.size();
1871 if (NumElts == 0)
1872 return SDValue(); // empty struct
1874 for (unsigned i = 0; i != NumElts; ++i) {
1875 EVT EltVT = ValueVTs[i];
1876 if (isa<UndefValue>(C))
1877 Constants[i] = DAG.getUNDEF(EltVT);
1878 else if (EltVT.isFloatingPoint())
1879 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1880 else
1881 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1882 }
1883
1885 }
1886
1887 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1888 return DAG.getBlockAddress(BA, VT);
1889
1890 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1891 return getValue(Equiv->getGlobalValue());
1892
1893 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1894 return getValue(NC->getGlobalValue());
1895
1896 if (VT == MVT::aarch64svcount) {
1897 assert(C->isNullValue() && "Can only zero this target type!");
1898 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1899 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1900 }
1901
1902 VectorType *VecTy = cast<VectorType>(V->getType());
1903
1904 // Now that we know the number and type of the elements, get that number of
1905 // elements into the Ops array based on what kind of constant it is.
1906 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1908 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1909 for (unsigned i = 0; i != NumElements; ++i)
1910 Ops.push_back(getValue(CV->getOperand(i)));
1911
1912 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1913 }
1914
1915 if (isa<ConstantAggregateZero>(C)) {
1916 EVT EltVT =
1918
1919 SDValue Op;
1920 if (EltVT.isFloatingPoint())
1921 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1922 else
1923 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1924
1925 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1926 }
1927
1928 llvm_unreachable("Unknown vector constant");
1929 }
1930
1931 // If this is a static alloca, generate it as the frameindex instead of
1932 // computation.
1933 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1935 FuncInfo.StaticAllocaMap.find(AI);
1936 if (SI != FuncInfo.StaticAllocaMap.end())
1937 return DAG.getFrameIndex(
1938 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1939 }
1940
1941 // If this is an instruction which fast-isel has deferred, select it now.
1942 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1944
1945 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1946 Inst->getType(), std::nullopt);
1947 SDValue Chain = DAG.getEntryNode();
1948 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1949 }
1950
1951 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1952 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1953
1954 if (const auto *BB = dyn_cast<BasicBlock>(V))
1955 return DAG.getBasicBlock(FuncInfo.getMBB(BB));
1956
1957 llvm_unreachable("Can't get register for value!");
1958}
1959
1960void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1962 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1963 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1964 bool IsSEH = isAsynchronousEHPersonality(Pers);
1965 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1966 if (!IsSEH)
1967 CatchPadMBB->setIsEHScopeEntry();
1968 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1969 if (IsMSVCCXX || IsCoreCLR)
1970 CatchPadMBB->setIsEHFuncletEntry();
1971}
1972
1973void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1974 // Update machine-CFG edge.
1975 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());
1976 FuncInfo.MBB->addSuccessor(TargetMBB);
1977 TargetMBB->setIsEHCatchretTarget(true);
1979
1981 bool IsSEH = isAsynchronousEHPersonality(Pers);
1982 if (IsSEH) {
1983 // If this is not a fall-through branch or optimizations are switched off,
1984 // emit the branch.
1985 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1987 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1988 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1989 return;
1990 }
1991
1992 // Figure out the funclet membership for the catchret's successor.
1993 // This will be used by the FuncletLayout pass to determine how to order the
1994 // BB's.
1995 // A 'catchret' returns to the outer scope's color.
1996 Value *ParentPad = I.getCatchSwitchParentPad();
1997 const BasicBlock *SuccessorColor;
1998 if (isa<ConstantTokenNone>(ParentPad))
1999 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2000 else
2001 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
2002 assert(SuccessorColor && "No parent funclet for catchret!");
2003 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(SuccessorColor);
2004 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2005
2006 // Create the terminator node.
2008 getControlRoot(), DAG.getBasicBlock(TargetMBB),
2009 DAG.getBasicBlock(SuccessorColorMBB));
2010 DAG.setRoot(Ret);
2011}
2012
2013void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2014 // Don't emit any special code for the cleanuppad instruction. It just marks
2015 // the start of an EH scope/funclet.
2018 if (Pers != EHPersonality::Wasm_CXX) {
2021 }
2022}
2023
2024// In wasm EH, even though a catchpad may not catch an exception if a tag does
2025// not match, it is OK to add only the first unwind destination catchpad to the
2026// successors, because there will be at least one invoke instruction within the
2027// catch scope that points to the next unwind destination, if one exists, so
2028// CFGSort cannot mess up with BB sorting order.
2029// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
2030// call within them, and catchpads only consisting of 'catch (...)' have a
2031// '__cxa_end_catch' call within them, both of which generate invokes in case
2032// the next unwind destination exists, i.e., the next unwind destination is not
2033// the caller.)
2034//
2035// Having at most one EH pad successor is also simpler and helps later
2036// transformations.
2037//
2038// For example,
2039// current:
2040// invoke void @foo to ... unwind label %catch.dispatch
2041// catch.dispatch:
2042// %0 = catchswitch within ... [label %catch.start] unwind label %next
2043// catch.start:
2044// ...
2045// ... in this BB or some other child BB dominated by this BB there will be an
2046// invoke that points to 'next' BB as an unwind destination
2047//
2048// next: ; We don't need to add this to 'current' BB's successor
2049// ...
2051 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2052 BranchProbability Prob,
2053 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2054 &UnwindDests) {
2055 while (EHPadBB) {
2056 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2057 if (isa<CleanupPadInst>(Pad)) {
2058 // Stop on cleanup pads.
2059 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2060 UnwindDests.back().first->setIsEHScopeEntry();
2061 break;
2062 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2063 // Add the catchpad handlers to the possible destinations. We don't
2064 // continue to the unwind destination of the catchswitch for wasm.
2065 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2066 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2067 UnwindDests.back().first->setIsEHScopeEntry();
2068 }
2069 break;
2070 } else {
2071 continue;
2072 }
2073 }
2074}
2075
2076/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2077/// many places it could ultimately go. In the IR, we have a single unwind
2078/// destination, but in the machine CFG, we enumerate all the possible blocks.
2079/// This function skips over imaginary basic blocks that hold catchswitch
2080/// instructions, and finds all the "real" machine
2081/// basic block destinations. As those destinations may not be successors of
2082/// EHPadBB, here we also calculate the edge probability to those destinations.
2083/// The passed-in Prob is the edge probability to EHPadBB.
2085 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2086 BranchProbability Prob,
2087 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2088 &UnwindDests) {
2089 EHPersonality Personality =
2091 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2092 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2093 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2094 bool IsSEH = isAsynchronousEHPersonality(Personality);
2095
2096 if (IsWasmCXX) {
2097 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
2098 assert(UnwindDests.size() <= 1 &&
2099 "There should be at most one unwind destination for wasm");
2100 return;
2101 }
2102
2103 while (EHPadBB) {
2104 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2105 BasicBlock *NewEHPadBB = nullptr;
2106 if (isa<LandingPadInst>(Pad)) {
2107 // Stop on landingpads. They are not funclets.
2108 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2109 break;
2110 } else if (isa<CleanupPadInst>(Pad)) {
2111 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2112 // personalities.
2113 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2114 UnwindDests.back().first->setIsEHScopeEntry();
2115 UnwindDests.back().first->setIsEHFuncletEntry();
2116 break;
2117 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2118 // Add the catchpad handlers to the possible destinations.
2119 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2120 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2121 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2122 if (IsMSVCCXX || IsCoreCLR)
2123 UnwindDests.back().first->setIsEHFuncletEntry();
2124 if (!IsSEH)
2125 UnwindDests.back().first->setIsEHScopeEntry();
2126 }
2127 NewEHPadBB = CatchSwitch->getUnwindDest();
2128 } else {
2129 continue;
2130 }
2131
2132 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2133 if (BPI && NewEHPadBB)
2134 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2135 EHPadBB = NewEHPadBB;
2136 }
2137}
2138
2139void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2140 // Update successor info.
2142 auto UnwindDest = I.getUnwindDest();
2144 BranchProbability UnwindDestProb =
2145 (BPI && UnwindDest)
2146 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2148 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2149 for (auto &UnwindDest : UnwindDests) {
2150 UnwindDest.first->setIsEHPad();
2151 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2152 }
2154
2155 // Create the terminator node.
2156 SDValue Ret =
2158 DAG.setRoot(Ret);
2159}
2160
2161void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2162 report_fatal_error("visitCatchSwitch not yet implemented!");
2163}
2164
2165void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2167 auto &DL = DAG.getDataLayout();
2168 SDValue Chain = getControlRoot();
2171
2172 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2173 // lower
2174 //
2175 // %val = call <ty> @llvm.experimental.deoptimize()
2176 // ret <ty> %val
2177 //
2178 // differently.
2179 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2181 return;
2182 }
2183
2184 if (!FuncInfo.CanLowerReturn) {
2185 unsigned DemoteReg = FuncInfo.DemoteRegister;
2186 const Function *F = I.getParent()->getParent();
2187
2188 // Emit a store of the return value through the virtual register.
2189 // Leave Outs empty so that LowerReturn won't try to load return
2190 // registers the usual way.
2191 SmallVector<EVT, 1> PtrValueVTs;
2192 ComputeValueVTs(TLI, DL,
2193 PointerType::get(F->getContext(),
2195 PtrValueVTs);
2196
2197 SDValue RetPtr =
2198 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2199 SDValue RetOp = getValue(I.getOperand(0));
2200
2201 SmallVector<EVT, 4> ValueVTs, MemVTs;
2203 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2204 &Offsets, 0);
2205 unsigned NumValues = ValueVTs.size();
2206
2207 SmallVector<SDValue, 4> Chains(NumValues);
2208 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2209 for (unsigned i = 0; i != NumValues; ++i) {
2210 // An aggregate return value cannot wrap around the address space, so
2211 // offsets to its parts don't wrap either.
2213 TypeSize::getFixed(Offsets[i]));
2214
2215 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2216 if (MemVTs[i] != ValueVTs[i])
2217 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2218 Chains[i] = DAG.getStore(
2219 Chain, getCurSDLoc(), Val,
2220 // FIXME: better loc info would be nice.
2222 commonAlignment(BaseAlign, Offsets[i]));
2223 }
2224
2226 MVT::Other, Chains);
2227 } else if (I.getNumOperands() != 0) {
2228 SmallVector<EVT, 4> ValueVTs;
2229 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2230 unsigned NumValues = ValueVTs.size();
2231 if (NumValues) {
2232 SDValue RetOp = getValue(I.getOperand(0));
2233
2234 const Function *F = I.getParent()->getParent();
2235
2236 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2237 I.getOperand(0)->getType(), F->getCallingConv(),
2238 /*IsVarArg*/ false, DL);
2239
2240 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2241 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2242 ExtendKind = ISD::SIGN_EXTEND;
2243 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2244 ExtendKind = ISD::ZERO_EXTEND;
2245
2246 LLVMContext &Context = F->getContext();
2247 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2248
2249 for (unsigned j = 0; j != NumValues; ++j) {
2250 EVT VT = ValueVTs[j];
2251
2252 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2253 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2254
2255 CallingConv::ID CC = F->getCallingConv();
2256
2257 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2258 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2259 SmallVector<SDValue, 4> Parts(NumParts);
2261 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2262 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2263
2264 // 'inreg' on function refers to return value
2266 if (RetInReg)
2267 Flags.setInReg();
2268
2269 if (I.getOperand(0)->getType()->isPointerTy()) {
2270 Flags.setPointer();
2271 Flags.setPointerAddrSpace(
2272 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2273 }
2274
2275 if (NeedsRegBlock) {
2276 Flags.setInConsecutiveRegs();
2277 if (j == NumValues - 1)
2278 Flags.setInConsecutiveRegsLast();
2279 }
2280
2281 // Propagate extension type if any
2282 if (ExtendKind == ISD::SIGN_EXTEND)
2283 Flags.setSExt();
2284 else if (ExtendKind == ISD::ZERO_EXTEND)
2285 Flags.setZExt();
2286
2287 for (unsigned i = 0; i < NumParts; ++i) {
2288 Outs.push_back(ISD::OutputArg(Flags,
2289 Parts[i].getValueType().getSimpleVT(),
2290 VT, /*isfixed=*/true, 0, 0));
2291 OutVals.push_back(Parts[i]);
2292 }
2293 }
2294 }
2295 }
2296
2297 // Push in swifterror virtual register as the last element of Outs. This makes
2298 // sure swifterror virtual register will be returned in the swifterror
2299 // physical register.
2300 const Function *F = I.getParent()->getParent();
2301 if (TLI.supportSwiftError() &&
2302 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2303 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2305 Flags.setSwiftError();
2307 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2308 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2309 // Create SDNode for the swifterror virtual register.
2310 OutVals.push_back(
2313 EVT(TLI.getPointerTy(DL))));
2314 }
2315
2316 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2317 CallingConv::ID CallConv =
2320 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2321
2322 // Verify that the target's LowerReturn behaved as expected.
2323 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2324 "LowerReturn didn't return a valid chain!");
2325
2326 // Update the DAG with the new chain value resulting from return lowering.
2327 DAG.setRoot(Chain);
2328}
2329
2330/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2331/// created for it, emit nodes to copy the value into the virtual
2332/// registers.
2334 // Skip empty types
2335 if (V->getType()->isEmptyTy())
2336 return;
2337
2339 if (VMI != FuncInfo.ValueMap.end()) {
2340 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2341 "Unused value assigned virtual registers!");
2342 CopyValueToVirtualRegister(V, VMI->second);
2343 }
2344}
2345
2346/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2347/// the current basic block, add it to ValueMap now so that we'll get a
2348/// CopyTo/FromReg.
2350 // No need to export constants.
2351 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2352
2353 // Already exported?
2354 if (FuncInfo.isExportedInst(V)) return;
2355
2358}
2359
2361 const BasicBlock *FromBB) {
2362 // The operands of the setcc have to be in this block. We don't know
2363 // how to export them from some other block.
2364 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2365 // Can export from current BB.
2366 if (VI->getParent() == FromBB)
2367 return true;
2368
2369 // Is already exported, noop.
2370 return FuncInfo.isExportedInst(V);
2371 }
2372
2373 // If this is an argument, we can export it if the BB is the entry block or
2374 // if it is already exported.
2375 if (isa<Argument>(V)) {
2376 if (FromBB->isEntryBlock())
2377 return true;
2378
2379 // Otherwise, can only export this if it is already exported.
2380 return FuncInfo.isExportedInst(V);
2381 }
2382
2383 // Otherwise, constants can always be exported.
2384 return true;
2385}
2386
2387/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2389SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2390 const MachineBasicBlock *Dst) const {
2392 const BasicBlock *SrcBB = Src->getBasicBlock();
2393 const BasicBlock *DstBB = Dst->getBasicBlock();
2394 if (!BPI) {
2395 // If BPI is not available, set the default probability as 1 / N, where N is
2396 // the number of successors.
2397 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2398 return BranchProbability(1, SuccSize);
2399 }
2400 return BPI->getEdgeProbability(SrcBB, DstBB);
2401}
2402
2403void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2404 MachineBasicBlock *Dst,
2405 BranchProbability Prob) {
2406 if (!FuncInfo.BPI)
2407 Src->addSuccessorWithoutProb(Dst);
2408 else {
2409 if (Prob.isUnknown())
2410 Prob = getEdgeProbability(Src, Dst);
2411 Src->addSuccessor(Dst, Prob);
2412 }
2413}
2414
2415static bool InBlock(const Value *V, const BasicBlock *BB) {
2416 if (const Instruction *I = dyn_cast<Instruction>(V))
2417 return I->getParent() == BB;
2418 return true;
2419}
2420
2421/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2422/// This function emits a branch and is used at the leaves of an OR or an
2423/// AND operator tree.
2424void
2427 MachineBasicBlock *FBB,
2428 MachineBasicBlock *CurBB,
2429 MachineBasicBlock *SwitchBB,
2430 BranchProbability TProb,
2431 BranchProbability FProb,
2432 bool InvertCond) {
2433 const BasicBlock *BB = CurBB->getBasicBlock();
2434
2435 // If the leaf of the tree is a comparison, merge the condition into
2436 // the caseblock.
2437 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2438 // The operands of the cmp have to be in this block. We don't know
2439 // how to export them from some other block. If this is the first block
2440 // of the sequence, no exporting is needed.
2441 if (CurBB == SwitchBB ||
2442 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2443 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2444 ISD::CondCode Condition;
2445 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2446 ICmpInst::Predicate Pred =
2447 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2448 Condition = getICmpCondCode(Pred);
2449 } else {
2450 const FCmpInst *FC = cast<FCmpInst>(Cond);
2451 FCmpInst::Predicate Pred =
2452 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2453 Condition = getFCmpCondCode(Pred);
2454 if (TM.Options.NoNaNsFPMath)
2455 Condition = getFCmpCodeWithoutNaN(Condition);
2456 }
2457
2458 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2459 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2460 SL->SwitchCases.push_back(CB);
2461 return;
2462 }
2463 }
2464
2465 // Create a CaseBlock record representing this branch.
2466 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2468 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2469 SL->SwitchCases.push_back(CB);
2470}
2471
2472// Collect dependencies on V recursively. This is used for the cost analysis in
2473// `shouldKeepJumpConditionsTogether`.
2477 unsigned Depth = 0) {
2478 // Return false if we have an incomplete count.
2480 return false;
2481
2482 auto *I = dyn_cast<Instruction>(V);
2483 if (I == nullptr)
2484 return true;
2485
2486 if (Necessary != nullptr) {
2487 // This instruction is necessary for the other side of the condition so
2488 // don't count it.
2489 if (Necessary->contains(I))
2490 return true;
2491 }
2492
2493 // Already added this dep.
2494 if (!Deps->try_emplace(I, false).second)
2495 return true;
2496
2497 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2498 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2499 Depth + 1))
2500 return false;
2501 return true;
2502}
2503
2505 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,
2506 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2508 if (I.getNumSuccessors() != 2)
2509 return false;
2510
2511 if (!I.isConditional())
2512 return false;
2513
2514 if (Params.BaseCost < 0)
2515 return false;
2516
2517 // Baseline cost.
2518 InstructionCost CostThresh = Params.BaseCost;
2519
2520 BranchProbabilityInfo *BPI = nullptr;
2521 if (Params.LikelyBias || Params.UnlikelyBias)
2522 BPI = FuncInfo.BPI;
2523 if (BPI != nullptr) {
2524 // See if we are either likely to get an early out or compute both lhs/rhs
2525 // of the condition.
2526 BasicBlock *IfFalse = I.getSuccessor(0);
2527 BasicBlock *IfTrue = I.getSuccessor(1);
2528
2529 std::optional<bool> Likely;
2530 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2531 Likely = true;
2532 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2533 Likely = false;
2534
2535 if (Likely) {
2536 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2537 // Its likely we will have to compute both lhs and rhs of condition
2538 CostThresh += Params.LikelyBias;
2539 else {
2540 if (Params.UnlikelyBias < 0)
2541 return false;
2542 // Its likely we will get an early out.
2543 CostThresh -= Params.UnlikelyBias;
2544 }
2545 }
2546 }
2547
2548 if (CostThresh <= 0)
2549 return false;
2550
2551 // Collect "all" instructions that lhs condition is dependent on.
2552 // Use map for stable iteration (to avoid non-determanism of iteration of
2553 // SmallPtrSet). The `bool` value is just a dummy.
2555 collectInstructionDeps(&LhsDeps, Lhs);
2556 // Collect "all" instructions that rhs condition is dependent on AND are
2557 // dependencies of lhs. This gives us an estimate on which instructions we
2558 // stand to save by splitting the condition.
2559 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2560 return false;
2561 // Add the compare instruction itself unless its a dependency on the LHS.
2562 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2563 if (!LhsDeps.contains(RhsI))
2564 RhsDeps.try_emplace(RhsI, false);
2565
2566 const auto &TLI = DAG.getTargetLoweringInfo();
2567 const auto &TTI =
2568 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
2569
2570 InstructionCost CostOfIncluding = 0;
2571 // See if this instruction will need to computed independently of whether RHS
2572 // is.
2573 Value *BrCond = I.getCondition();
2574 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2575 for (const auto *U : Ins->users()) {
2576 // If user is independent of RHS calculation we don't need to count it.
2577 if (auto *UIns = dyn_cast<Instruction>(U))
2578 if (UIns != BrCond && !RhsDeps.contains(UIns))
2579 return false;
2580 }
2581 return true;
2582 };
2583
2584 // Prune instructions from RHS Deps that are dependencies of unrelated
2585 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2586 // arbitrary and just meant to cap the how much time we spend in the pruning
2587 // loop. Its highly unlikely to come into affect.
2588 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2589 // Stop after a certain point. No incorrectness from including too many
2590 // instructions.
2591 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2592 const Instruction *ToDrop = nullptr;
2593 for (const auto &InsPair : RhsDeps) {
2594 if (!ShouldCountInsn(InsPair.first)) {
2595 ToDrop = InsPair.first;
2596 break;
2597 }
2598 }
2599 if (ToDrop == nullptr)
2600 break;
2601 RhsDeps.erase(ToDrop);
2602 }
2603
2604 for (const auto &InsPair : RhsDeps) {
2605 // Finally accumulate latency that we can only attribute to computing the
2606 // RHS condition. Use latency because we are essentially trying to calculate
2607 // the cost of the dependency chain.
2608 // Possible TODO: We could try to estimate ILP and make this more precise.
2609 CostOfIncluding +=
2611
2612 if (CostOfIncluding > CostThresh)
2613 return false;
2614 }
2615 return true;
2616}
2617
2620 MachineBasicBlock *FBB,
2621 MachineBasicBlock *CurBB,
2622 MachineBasicBlock *SwitchBB,
2624 BranchProbability TProb,
2625 BranchProbability FProb,
2626 bool InvertCond) {
2627 // Skip over not part of the tree and remember to invert op and operands at
2628 // next level.
2629 Value *NotCond;
2630 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2631 InBlock(NotCond, CurBB->getBasicBlock())) {
2632 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2633 !InvertCond);
2634 return;
2635 }
2636
2637 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2638 const Value *BOpOp0, *BOpOp1;
2639 // Compute the effective opcode for Cond, taking into account whether it needs
2640 // to be inverted, e.g.
2641 // and (not (or A, B)), C
2642 // gets lowered as
2643 // and (and (not A, not B), C)
2645 if (BOp) {
2646 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2647 ? Instruction::And
2648 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2649 ? Instruction::Or
2651 if (InvertCond) {
2652 if (BOpc == Instruction::And)
2653 BOpc = Instruction::Or;
2654 else if (BOpc == Instruction::Or)
2655 BOpc = Instruction::And;
2656 }
2657 }
2658
2659 // If this node is not part of the or/and tree, emit it as a branch.
2660 // Note that all nodes in the tree should have same opcode.
2661 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2662 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2663 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2664 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2665 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2666 TProb, FProb, InvertCond);
2667 return;
2668 }
2669
2670 // Create TmpBB after CurBB.
2671 MachineFunction::iterator BBI(CurBB);
2674 CurBB->getParent()->insert(++BBI, TmpBB);
2675
2676 if (Opc == Instruction::Or) {
2677 // Codegen X | Y as:
2678 // BB1:
2679 // jmp_if_X TBB
2680 // jmp TmpBB
2681 // TmpBB:
2682 // jmp_if_Y TBB
2683 // jmp FBB
2684 //
2685
2686 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2687 // The requirement is that
2688 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2689 // = TrueProb for original BB.
2690 // Assuming the original probabilities are A and B, one choice is to set
2691 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2692 // A/(1+B) and 2B/(1+B). This choice assumes that
2693 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2694 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2695 // TmpBB, but the math is more complicated.
2696
2697 auto NewTrueProb = TProb / 2;
2698 auto NewFalseProb = TProb / 2 + FProb;
2699 // Emit the LHS condition.
2700 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2701 NewFalseProb, InvertCond);
2702
2703 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2704 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2705 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2706 // Emit the RHS condition into TmpBB.
2707 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2708 Probs[1], InvertCond);
2709 } else {
2710 assert(Opc == Instruction::And && "Unknown merge op!");
2711 // Codegen X & Y as:
2712 // BB1:
2713 // jmp_if_X TmpBB
2714 // jmp FBB
2715 // TmpBB:
2716 // jmp_if_Y TBB
2717 // jmp FBB
2718 //
2719 // This requires creation of TmpBB after CurBB.
2720
2721 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2722 // The requirement is that
2723 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2724 // = FalseProb for original BB.
2725 // Assuming the original probabilities are A and B, one choice is to set
2726 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2727 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2728 // TrueProb for BB1 * FalseProb for TmpBB.
2729
2730 auto NewTrueProb = TProb + FProb / 2;
2731 auto NewFalseProb = FProb / 2;
2732 // Emit the LHS condition.
2733 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2734 NewFalseProb, InvertCond);
2735
2736 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2737 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2738 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2739 // Emit the RHS condition into TmpBB.
2740 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2741 Probs[1], InvertCond);
2742 }
2743}
2744
2745/// If the set of cases should be emitted as a series of branches, return true.
2746/// If we should emit this as a bunch of and/or'd together conditions, return
2747/// false.
2748bool
2749SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2750 if (Cases.size() != 2) return true;
2751
2752 // If this is two comparisons of the same values or'd or and'd together, they
2753 // will get folded into a single comparison, so don't emit two blocks.
2754 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2755 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2756 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2757 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2758 return false;
2759 }
2760
2761 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2762 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2763 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2764 Cases[0].CC == Cases[1].CC &&
2765 isa<Constant>(Cases[0].CmpRHS) &&
2766 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2767 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2768 return false;
2769 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2770 return false;
2771 }
2772
2773 return true;
2774}
2775
2776void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2778
2779 // Update machine-CFG edges.
2780 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2781
2782 if (I.isUnconditional()) {
2783 // Update machine-CFG edges.
2784 BrMBB->addSuccessor(Succ0MBB);
2785
2786 // If this is not a fall-through branch or optimizations are switched off,
2787 // emit the branch.
2788 if (Succ0MBB != NextBlock(BrMBB) ||
2790 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2791 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2792 setValue(&I, Br);
2793 DAG.setRoot(Br);
2794 }
2795
2796 return;
2797 }
2798
2799 // If this condition is one of the special cases we handle, do special stuff
2800 // now.
2801 const Value *CondVal = I.getCondition();
2802 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(I.getSuccessor(1));
2803
2804 // If this is a series of conditions that are or'd or and'd together, emit
2805 // this as a sequence of branches instead of setcc's with and/or operations.
2806 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2807 // unpredictable branches, and vector extracts because those jumps are likely
2808 // expensive for any target), this should improve performance.
2809 // For example, instead of something like:
2810 // cmp A, B
2811 // C = seteq
2812 // cmp D, E
2813 // F = setle
2814 // or C, F
2815 // jnz foo
2816 // Emit:
2817 // cmp A, B
2818 // je foo
2819 // cmp D, E
2820 // jle foo
2821 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2822 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2823 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2824 Value *Vec;
2825 const Value *BOp0, *BOp1;
2827 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2828 Opcode = Instruction::And;
2829 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2830 Opcode = Instruction::Or;
2831
2832 if (Opcode &&
2833 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2834 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2836 FuncInfo, I, Opcode, BOp0, BOp1,
2838 Opcode, BOp0, BOp1))) {
2839 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2840 getEdgeProbability(BrMBB, Succ0MBB),
2841 getEdgeProbability(BrMBB, Succ1MBB),
2842 /*InvertCond=*/false);
2843 // If the compares in later blocks need to use values not currently
2844 // exported from this block, export them now. This block should always
2845 // be the first entry.
2846 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2847
2848 // Allow some cases to be rejected.
2849 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2850 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2851 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2852 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2853 }
2854
2855 // Emit the branch for this block.
2856 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2857 SL->SwitchCases.erase(SL->SwitchCases.begin());
2858 return;
2859 }
2860
2861 // Okay, we decided not to do this, remove any inserted MBB's and clear
2862 // SwitchCases.
2863 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2864 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2865
2866 SL->SwitchCases.clear();
2867 }
2868 }
2869
2870 // Create a CaseBlock record representing this branch.
2872 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2873
2874 // Use visitSwitchCase to actually insert the fast branch sequence for this
2875 // cond branch.
2876 visitSwitchCase(CB, BrMBB);
2877}
2878
2879/// visitSwitchCase - Emits the necessary code to represent a single node in
2880/// the binary search tree resulting from lowering a switch instruction.
2882 MachineBasicBlock *SwitchBB) {
2883 SDValue Cond;
2884 SDValue CondLHS = getValue(CB.CmpLHS);
2885 SDLoc dl = CB.DL;
2886
2887 if (CB.CC == ISD::SETTRUE) {
2888 // Branch or fall through to TrueBB.
2889 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2890 SwitchBB->normalizeSuccProbs();
2891 if (CB.TrueBB != NextBlock(SwitchBB)) {
2892 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2893 DAG.getBasicBlock(CB.TrueBB)));
2894 }
2895 return;
2896 }
2897
2898 auto &TLI = DAG.getTargetLoweringInfo();
2899 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2900
2901 // Build the setcc now.
2902 if (!CB.CmpMHS) {
2903 // Fold "(X == true)" to X and "(X == false)" to !X to
2904 // handle common cases produced by branch lowering.
2905 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2906 CB.CC == ISD::SETEQ)
2907 Cond = CondLHS;
2908 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2909 CB.CC == ISD::SETEQ) {
2910 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2911 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2912 } else {
2913 SDValue CondRHS = getValue(CB.CmpRHS);
2914
2915 // If a pointer's DAG type is larger than its memory type then the DAG
2916 // values are zero-extended. This breaks signed comparisons so truncate
2917 // back to the underlying type before doing the compare.
2918 if (CondLHS.getValueType() != MemVT) {
2919 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2920 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2921 }
2922 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2923 }
2924 } else {
2925 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2926
2927 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2928 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2929
2930 SDValue CmpOp = getValue(CB.CmpMHS);
2931 EVT VT = CmpOp.getValueType();
2932
2933 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2934 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2935 ISD::SETLE);
2936 } else {
2937 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2938 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2939 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2940 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2941 }
2942 }
2943
2944 // Update successor info
2945 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2946 // TrueBB and FalseBB are always different unless the incoming IR is
2947 // degenerate. This only happens when running llc on weird IR.
2948 if (CB.TrueBB != CB.FalseBB)
2949 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2950 SwitchBB->normalizeSuccProbs();
2951
2952 // If the lhs block is the next block, invert the condition so that we can
2953 // fall through to the lhs instead of the rhs block.
2954 if (CB.TrueBB == NextBlock(SwitchBB)) {
2955 std::swap(CB.TrueBB, CB.FalseBB);
2956 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2957 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2958 }
2959
2960 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2961 MVT::Other, getControlRoot(), Cond,
2963
2964 setValue(CurInst, BrCond);
2965
2966 // Insert the false branch. Do this even if it's a fall through branch,
2967 // this makes it easier to do DAG optimizations which require inverting
2968 // the branch condition.
2969 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2971
2972 DAG.setRoot(BrCond);
2973}
2974
2975/// visitJumpTable - Emit JumpTable node in the current MBB
2977 // Emit the code for the jump table
2978 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2979 assert(JT.Reg != -1U && "Should lower JT Header first!");
2981 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
2982 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2983 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2984 Index.getValue(1), Table, Index);
2985 DAG.setRoot(BrJumpTable);
2986}
2987
2988/// visitJumpTableHeader - This function emits necessary code to produce index
2989/// in the JumpTable from switch case.
2991 JumpTableHeader &JTH,
2992 MachineBasicBlock *SwitchBB) {
2993 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2994 const SDLoc &dl = *JT.SL;
2995
2996 // Subtract the lowest switch case value from the value being switched on.
2997 SDValue SwitchOp = getValue(JTH.SValue);
2998 EVT VT = SwitchOp.getValueType();
2999 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
3000 DAG.getConstant(JTH.First, dl, VT));
3001
3002 // The SDNode we just created, which holds the value being switched on minus
3003 // the smallest case value, needs to be copied to a virtual register so it
3004 // can be used as an index into the jump table in a subsequent basic block.
3005 // This value may be smaller or larger than the target's pointer type, and
3006 // therefore require extension or truncating.
3008 SwitchOp =
3010
3011 unsigned JumpTableReg =
3013 SDValue CopyTo =
3014 DAG.getCopyToReg(getControlRoot(), dl, JumpTableReg, SwitchOp);
3015 JT.Reg = JumpTableReg;
3016
3017 if (!JTH.FallthroughUnreachable) {
3018 // Emit the range check for the jump table, and branch to the default block
3019 // for the switch statement if the value being switched on exceeds the
3020 // largest case in the switch.
3021 SDValue CMP = DAG.getSetCC(
3023 Sub.getValueType()),
3024 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3025
3026 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3027 MVT::Other, CopyTo, CMP,
3028 DAG.getBasicBlock(JT.Default));
3029
3030 // Avoid emitting unnecessary branches to the next block.
3031 if (JT.MBB != NextBlock(SwitchBB))
3032 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3033 DAG.getBasicBlock(JT.MBB));
3034
3035 DAG.setRoot(BrCond);
3036 } else {
3037 // Avoid emitting unnecessary branches to the next block.
3038 if (JT.MBB != NextBlock(SwitchBB))
3039 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3040 DAG.getBasicBlock(JT.MBB)));
3041 else
3042 DAG.setRoot(CopyTo);
3043 }
3044}
3045
3046/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3047/// variable if there exists one.
3049 SDValue &Chain) {
3050 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3051 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3052 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3056 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3057 if (Global) {
3058 MachinePointerInfo MPInfo(Global);
3062 MPInfo, Flags, LocationSize::precise(PtrTy.getSizeInBits() / 8),
3063 DAG.getEVTAlign(PtrTy));
3064 DAG.setNodeMemRefs(Node, {MemRef});
3065 }
3066 if (PtrTy != PtrMemTy)
3067 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3068 return SDValue(Node, 0);
3069}
3070
3071/// Codegen a new tail for a stack protector check ParentMBB which has had its
3072/// tail spliced into a stack protector check success bb.
3073///
3074/// For a high level explanation of how this fits into the stack protector
3075/// generation see the comment on the declaration of class
3076/// StackProtectorDescriptor.
3078 MachineBasicBlock *ParentBB) {
3079
3080 // First create the loads to the guard/stack slot for the comparison.
3082 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3083 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3084
3085 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3086 int FI = MFI.getStackProtectorIndex();
3087
3088 SDValue Guard;
3089 SDLoc dl = getCurSDLoc();
3090 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3091 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3092 Align Align =
3093 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
3094
3095 // Generate code to load the content of the guard slot.
3096 SDValue GuardVal = DAG.getLoad(
3097 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3100
3101 if (TLI.useStackGuardXorFP())
3102 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3103
3104 // Retrieve guard check function, nullptr if instrumentation is inlined.
3105 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3106 // The target provides a guard check function to validate the guard value.
3107 // Generate a call to that function with the content of the guard slot as
3108 // argument.
3109 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3110 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3111
3114 Entry.Node = GuardVal;
3115 Entry.Ty = FnTy->getParamType(0);
3116 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3117 Entry.IsInReg = true;
3118 Args.push_back(Entry);
3119
3123 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3124 getValue(GuardCheckFn), std::move(Args));
3125
3126 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3127 DAG.setRoot(Result.second);
3128 return;
3129 }
3130
3131 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3132 // Otherwise, emit a volatile load to retrieve the stack guard value.
3133 SDValue Chain = DAG.getEntryNode();
3134 if (TLI.useLoadStackGuardNode()) {
3135 Guard = getLoadStackGuard(DAG, dl, Chain);
3136 } else {
3137 const Value *IRGuard = TLI.getSDagStackGuard(M);
3138 SDValue GuardPtr = getValue(IRGuard);
3139
3140 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3141 MachinePointerInfo(IRGuard, 0), Align,
3143 }
3144
3145 // Perform the comparison via a getsetcc.
3147 *DAG.getContext(),
3148 Guard.getValueType()),
3149 Guard, GuardVal, ISD::SETNE);
3150
3151 // If the guard/stackslot do not equal, branch to failure MBB.
3152 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3153 MVT::Other, GuardVal.getOperand(0),
3154 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3155 // Otherwise branch to success MBB.
3156 SDValue Br = DAG.getNode(ISD::BR, dl,
3157 MVT::Other, BrCond,
3159
3160 DAG.setRoot(Br);
3161}
3162
3163/// Codegen the failure basic block for a stack protector check.
3164///
3165/// A failure stack protector machine basic block consists simply of a call to
3166/// __stack_chk_fail().
3167///
3168/// For a high level explanation of how this fits into the stack protector
3169/// generation see the comment on the declaration of class
3170/// StackProtectorDescriptor.
3171void
3175 CallOptions.setDiscardResult(true);
3176 SDValue Chain =
3177 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3178 std::nullopt, CallOptions, getCurSDLoc())
3179 .second;
3180 // On PS4/PS5, the "return address" must still be within the calling
3181 // function, even if it's at the very end, so emit an explicit TRAP here.
3182 // Passing 'true' for doesNotReturn above won't generate the trap for us.
3183 if (TM.getTargetTriple().isPS())
3184 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3185 // WebAssembly needs an unreachable instruction after a non-returning call,
3186 // because the function return type can be different from __stack_chk_fail's
3187 // return type (void).
3188 if (TM.getTargetTriple().isWasm())
3189 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3190
3191 DAG.setRoot(Chain);
3192}
3193
3194/// visitBitTestHeader - This function emits necessary code to produce value
3195/// suitable for "bit tests"
3197 MachineBasicBlock *SwitchBB) {
3198 SDLoc dl = getCurSDLoc();
3199
3200 // Subtract the minimum value.
3201 SDValue SwitchOp = getValue(B.SValue);
3202 EVT VT = SwitchOp.getValueType();
3203 SDValue RangeSub =
3204 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3205
3206 // Determine the type of the test operands.
3208 bool UsePtrType = false;
3209 if (!TLI.isTypeLegal(VT)) {
3210 UsePtrType = true;
3211 } else {
3212 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
3213 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
3214 // Switch table case range are encoded into series of masks.
3215 // Just use pointer type, it's guaranteed to fit.
3216 UsePtrType = true;
3217 break;
3218 }
3219 }
3220 SDValue Sub = RangeSub;
3221 if (UsePtrType) {
3222 VT = TLI.getPointerTy(DAG.getDataLayout());
3223 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3224 }
3225
3226 B.RegVT = VT.getSimpleVT();
3227 B.Reg = FuncInfo.CreateReg(B.RegVT);
3228 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3229
3230 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3231
3232 if (!B.FallthroughUnreachable)
3233 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3234 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3235 SwitchBB->normalizeSuccProbs();
3236
3237 SDValue Root = CopyTo;
3238 if (!B.FallthroughUnreachable) {
3239 // Conditional branch to the default block.
3240 SDValue RangeCmp = DAG.getSetCC(dl,
3242 RangeSub.getValueType()),
3243 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3244 ISD::SETUGT);
3245
3246 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3247 DAG.getBasicBlock(B.Default));
3248 }
3249
3250 // Avoid emitting unnecessary branches to the next block.
3251 if (MBB != NextBlock(SwitchBB))
3252 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3253
3254 DAG.setRoot(Root);
3255}
3256
3257/// visitBitTestCase - this function produces one "bit test"
3259 MachineBasicBlock* NextMBB,
3260 BranchProbability BranchProbToNext,
3261 unsigned Reg,
3262 BitTestCase &B,
3263 MachineBasicBlock *SwitchBB) {
3264 SDLoc dl = getCurSDLoc();
3265 MVT VT = BB.RegVT;
3266 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3267 SDValue Cmp;
3268 unsigned PopCount = llvm::popcount(B.Mask);
3270 if (PopCount == 1) {
3271 // Testing for a single bit; just compare the shift count with what it
3272 // would need to be to shift a 1 bit in that position.
3273 Cmp = DAG.getSetCC(
3275 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3276 ISD::SETEQ);
3277 } else if (PopCount == BB.Range) {
3278 // There is only one zero bit in the range, test for it directly.
3279 Cmp = DAG.getSetCC(
3281 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3282 } else {
3283 // Make desired shift
3284 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3285 DAG.getConstant(1, dl, VT), ShiftOp);
3286
3287 // Emit bit tests and jumps
3288 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3289 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3290 Cmp = DAG.getSetCC(
3292 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3293 }
3294
3295 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3296 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3297 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3298 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3299 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3300 // one as they are relative probabilities (and thus work more like weights),
3301 // and hence we need to normalize them to let the sum of them become one.
3302 SwitchBB->normalizeSuccProbs();
3303
3304 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3305 MVT::Other, getControlRoot(),
3306 Cmp, DAG.getBasicBlock(B.TargetBB));
3307
3308 // Avoid emitting unnecessary branches to the next block.
3309 if (NextMBB != NextBlock(SwitchBB))
3310 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3311 DAG.getBasicBlock(NextMBB));
3312
3313 DAG.setRoot(BrAnd);
3314}
3315
3316void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3317 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3318
3319 // Retrieve successors. Look through artificial IR level blocks like
3320 // catchswitch for successors.
3321 MachineBasicBlock *Return = FuncInfo.getMBB(I.getSuccessor(0));
3322 const BasicBlock *EHPadBB = I.getSuccessor(1);
3323 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(EHPadBB);
3324
3325 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3326 // have to do anything here to lower funclet bundles.
3327 assert(!I.hasOperandBundlesOtherThan(
3328 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3329 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3330 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3331 LLVMContext::OB_clang_arc_attachedcall}) &&
3332 "Cannot lower invokes with arbitrary operand bundles yet!");
3333
3334 const Value *Callee(I.getCalledOperand());
3335 const Function *Fn = dyn_cast<Function>(Callee);
3336 if (isa<InlineAsm>(Callee))
3337 visitInlineAsm(I, EHPadBB);
3338 else if (Fn && Fn->isIntrinsic()) {
3339 switch (Fn->getIntrinsicID()) {
3340 default:
3341 llvm_unreachable("Cannot invoke this intrinsic");
3342 case Intrinsic::donothing:
3343 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3344 case Intrinsic::seh_try_begin:
3345 case Intrinsic::seh_scope_begin:
3346 case Intrinsic::seh_try_end:
3347 case Intrinsic::seh_scope_end:
3348 if (EHPadMBB)
3349 // a block referenced by EH table
3350 // so dtor-funclet not removed by opts
3351 EHPadMBB->setMachineBlockAddressTaken();
3352 break;
3353 case Intrinsic::experimental_patchpoint_void:
3354 case Intrinsic::experimental_patchpoint:
3355 visitPatchpoint(I, EHPadBB);
3356 break;
3357 case Intrinsic::experimental_gc_statepoint:
3358 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3359 break;
3360 case Intrinsic::wasm_rethrow: {
3361 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3362 // special because it can be invoked, so we manually lower it to a DAG
3363 // node here.
3365 Ops.push_back(getControlRoot()); // inchain for the terminator node
3367 Ops.push_back(
3368 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3370 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3372 break;
3373 }
3374 }
3375 } else if (I.hasDeoptState()) {
3376 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3377 // Eventually we will support lowering the @llvm.experimental.deoptimize
3378 // intrinsic, and right now there are no plans to support other intrinsics
3379 // with deopt state.
3380 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3381 } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
3382 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), EHPadBB);
3383 } else {
3384 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3385 }
3386
3387 // If the value of the invoke is used outside of its defining block, make it
3388 // available as a virtual register.
3389 // We already took care of the exported value for the statepoint instruction
3390 // during call to the LowerStatepoint.
3391 if (!isa<GCStatepointInst>(I)) {
3393 }
3394
3397 BranchProbability EHPadBBProb =
3398 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3400 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3401
3402 // Update successor info.
3403 addSuccessorWithProb(InvokeMBB, Return);
3404 for (auto &UnwindDest : UnwindDests) {
3405 UnwindDest.first->setIsEHPad();
3406 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3407 }
3408 InvokeMBB->normalizeSuccProbs();
3409
3410 // Drop into normal successor.
3412 DAG.getBasicBlock(Return)));
3413}
3414
3415void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3416 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3417
3418 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3419 // have to do anything here to lower funclet bundles.
3420 assert(!I.hasOperandBundlesOtherThan(
3421 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3422 "Cannot lower callbrs with arbitrary operand bundles yet!");
3423
3424 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3425 visitInlineAsm(I);
3427
3428 // Retrieve successors.
3430 Dests.insert(I.getDefaultDest());
3431 MachineBasicBlock *Return = FuncInfo.getMBB(I.getDefaultDest());
3432
3433 // Update successor info.
3434 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3435 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3436 BasicBlock *Dest = I.getIndirectDest(i);
3438 Target->setIsInlineAsmBrIndirectTarget();
3439 Target->setMachineBlockAddressTaken();
3440 Target->setLabelMustBeEmitted();
3441 // Don't add duplicate machine successors.
3442 if (Dests.insert(Dest).second)
3443 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3444 }
3445 CallBrMBB->normalizeSuccProbs();
3446
3447 // Drop into default successor.
3449 MVT::Other, getControlRoot(),
3450 DAG.getBasicBlock(Return)));
3451}
3452
3453void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3454 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3455}
3456
3457void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3459 "Call to landingpad not in landing pad!");
3460
3461 // If there aren't registers to copy the values into (e.g., during SjLj
3462 // exceptions), then don't bother to create these DAG nodes.
3464 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3465 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3466 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3467 return;
3468
3469 // If landingpad's return type is token type, we don't create DAG nodes
3470 // for its exception pointer and selector value. The extraction of exception
3471 // pointer or selector value from token type landingpads is not currently
3472 // supported.
3473 if (LP.getType()->isTokenTy())
3474 return;
3475
3476 SmallVector<EVT, 2> ValueVTs;
3477 SDLoc dl = getCurSDLoc();
3478 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3479 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3480
3481 // Get the two live-in registers as SDValues. The physregs have already been
3482 // copied into virtual registers.
3483 SDValue Ops[2];
3485 Ops[0] = DAG.getZExtOrTrunc(
3489 dl, ValueVTs[0]);
3490 } else {
3491 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3492 }
3493 Ops[1] = DAG.getZExtOrTrunc(
3497 dl, ValueVTs[1]);
3498
3499 // Merge into one.
3501 DAG.getVTList(ValueVTs), Ops);
3502 setValue(&LP, Res);
3503}
3504
3507 // Update JTCases.
3508 for (JumpTableBlock &JTB : SL->JTCases)
3509 if (JTB.first.HeaderBB == First)
3510 JTB.first.HeaderBB = Last;
3511
3512 // Update BitTestCases.
3513 for (BitTestBlock &BTB : SL->BitTestCases)
3514 if (BTB.Parent == First)
3515 BTB.Parent = Last;
3516}
3517
3518void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3519 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3520
3521 // Update machine-CFG edges with unique successors.
3523 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3524 BasicBlock *BB = I.getSuccessor(i);
3525 bool Inserted = Done.insert(BB).second;
3526 if (!Inserted)
3527 continue;
3528
3529 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3530 addSuccessorWithProb(IndirectBrMBB, Succ);
3531 }
3532 IndirectBrMBB->normalizeSuccProbs();
3533
3535 MVT::Other, getControlRoot(),
3536 getValue(I.getAddress())));
3537}
3538
3539void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3541 return;
3542
3543 // We may be able to ignore unreachable behind a noreturn call.
3544 if (const CallInst *Call = dyn_cast_or_null<CallInst>(I.getPrevNode());
3545 Call && Call->doesNotReturn()) {
3547 return;
3548 // Do not emit an additional trap instruction.
3549 if (Call->isNonContinuableTrap())
3550 return;
3551 }
3552
3553 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3554}
3555
3556void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3558 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3559 Flags.copyFMF(*FPOp);
3560
3561 SDValue Op = getValue(I.getOperand(0));
3562 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3563 Op, Flags);
3564 setValue(&I, UnNodeValue);
3565}
3566
3567void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3569 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3570 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3571 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3572 }
3573 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3574 Flags.setExact(ExactOp->isExact());
3575 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3576 Flags.setDisjoint(DisjointOp->isDisjoint());
3577 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3578 Flags.copyFMF(*FPOp);
3579
3580 SDValue Op1 = getValue(I.getOperand(0));
3581 SDValue Op2 = getValue(I.getOperand(1));
3582 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3583 Op1, Op2, Flags);
3584 setValue(&I, BinNodeValue);
3585}
3586
3587void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3588 SDValue Op1 = getValue(I.getOperand(0));
3589 SDValue Op2 = getValue(I.getOperand(1));
3590
3592 Op1.getValueType(), DAG.getDataLayout());
3593
3594 // Coerce the shift amount to the right type if we can. This exposes the
3595 // truncate or zext to optimization early.
3596 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3598 "Unexpected shift type");
3599 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3600 }
3601
3602 bool nuw = false;
3603 bool nsw = false;
3604 bool exact = false;
3605
3606 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3607
3608 if (const OverflowingBinaryOperator *OFBinOp =
3609 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3610 nuw = OFBinOp->hasNoUnsignedWrap();
3611 nsw = OFBinOp->hasNoSignedWrap();
3612 }
3613 if (const PossiblyExactOperator *ExactOp =
3614 dyn_cast<const PossiblyExactOperator>(&I))
3615 exact = ExactOp->isExact();
3616 }
3618 Flags.setExact(exact);
3619 Flags.setNoSignedWrap(nsw);
3620 Flags.setNoUnsignedWrap(nuw);
3621 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3622 Flags);
3623 setValue(&I, Res);
3624}
3625
3626void SelectionDAGBuilder::visitSDiv(const User &I) {
3627 SDValue Op1 = getValue(I.getOperand(0));
3628 SDValue Op2 = getValue(I.getOperand(1));
3629
3631 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3632 cast<PossiblyExactOperator>(&I)->isExact());
3634 Op2, Flags));
3635}
3636
3637void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3638 ICmpInst::Predicate predicate = I.getPredicate();
3639 SDValue Op1 = getValue(I.getOperand(0));
3640 SDValue Op2 = getValue(I.getOperand(1));
3641 ISD::CondCode Opcode = getICmpCondCode(predicate);
3642
3643 auto &TLI = DAG.getTargetLoweringInfo();
3644 EVT MemVT =
3645 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3646
3647 // If a pointer's DAG type is larger than its memory type then the DAG values
3648 // are zero-extended. This breaks signed comparisons so truncate back to the
3649 // underlying type before doing the compare.
3650 if (Op1.getValueType() != MemVT) {
3651 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3652 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3653 }
3654
3656 I.getType());
3657 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3658}
3659
3660void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3661 FCmpInst::Predicate predicate = I.getPredicate();
3662 SDValue Op1 = getValue(I.getOperand(0));
3663 SDValue Op2 = getValue(I.getOperand(1));
3664
3665 ISD::CondCode Condition = getFCmpCondCode(predicate);
3666 auto *FPMO = cast<FPMathOperator>(&I);
3667 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3668 Condition = getFCmpCodeWithoutNaN(Condition);
3669
3671 Flags.copyFMF(*FPMO);
3672 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3673
3675 I.getType());
3676 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3677}
3678
3679// Check if the condition of the select has one use or two users that are both
3680// selects with the same condition.
3681static bool hasOnlySelectUsers(const Value *Cond) {
3682 return llvm::all_of(Cond->users(), [](const Value *V) {
3683 return isa<SelectInst>(V);
3684 });
3685}
3686
3687void SelectionDAGBuilder::visitSelect(const User &I) {
3688 SmallVector<EVT, 4> ValueVTs;
3690 ValueVTs);
3691 unsigned NumValues = ValueVTs.size();
3692 if (NumValues == 0) return;
3693
3694 SmallVector<SDValue, 4> Values(NumValues);
3695 SDValue Cond = getValue(I.getOperand(0));
3696 SDValue LHSVal = getValue(I.getOperand(1));
3697 SDValue RHSVal = getValue(I.getOperand(2));
3698 SmallVector<SDValue, 1> BaseOps(1, Cond);
3700 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3701
3702 bool IsUnaryAbs = false;
3703 bool Negate = false;
3704
3706 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3707 Flags.copyFMF(*FPOp);
3708
3709 Flags.setUnpredictable(
3710 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3711
3712 // Min/max matching is only viable if all output VTs are the same.
3713 if (all_equal(ValueVTs)) {
3714 EVT VT = ValueVTs[0];
3715 LLVMContext &Ctx = *DAG.getContext();
3716 auto &TLI = DAG.getTargetLoweringInfo();
3717
3718 // We care about the legality of the operation after it has been type
3719 // legalized.
3720 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3721 VT = TLI.getTypeToTransformTo(Ctx, VT);
3722
3723 // If the vselect is legal, assume we want to leave this as a vector setcc +
3724 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3725 // min/max is legal on the scalar type.
3726 bool UseScalarMinMax = VT.isVector() &&
3728
3729 // ValueTracking's select pattern matching does not account for -0.0,
3730 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3731 // -0.0 is less than +0.0.
3732 const Value *LHS, *RHS;
3733 auto SPR = matchSelectPattern(&I, LHS, RHS);
3735 switch (SPR.Flavor) {
3736 case SPF_UMAX: Opc = ISD::UMAX; break;
3737 case SPF_UMIN: Opc = ISD::UMIN; break;
3738 case SPF_SMAX: Opc = ISD::SMAX; break;
3739 case SPF_SMIN: Opc = ISD::SMIN; break;
3740 case SPF_FMINNUM:
3741 switch (SPR.NaNBehavior) {
3742 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3743 case SPNB_RETURNS_NAN: break;
3744 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3745 case SPNB_RETURNS_ANY:
3747 (UseScalarMinMax &&
3749 Opc = ISD::FMINNUM;
3750 break;
3751 }
3752 break;
3753 case SPF_FMAXNUM:
3754 switch (SPR.NaNBehavior) {
3755 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3756 case SPNB_RETURNS_NAN: break;
3757 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3758 case SPNB_RETURNS_ANY:
3760 (UseScalarMinMax &&
3762 Opc = ISD::FMAXNUM;
3763 break;
3764 }
3765 break;
3766 case SPF_NABS:
3767 Negate = true;
3768 [[fallthrough]];
3769 case SPF_ABS:
3770 IsUnaryAbs = true;
3771 Opc = ISD::ABS;
3772 break;
3773 default: break;
3774 }
3775
3776 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3777 (TLI.isOperationLegalOrCustomOrPromote(Opc, VT) ||
3778 (UseScalarMinMax &&
3779 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3780 // If the underlying comparison instruction is used by any other
3781 // instruction, the consumed instructions won't be destroyed, so it is
3782 // not profitable to convert to a min/max.
3783 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3784 OpCode = Opc;
3785 LHSVal = getValue(LHS);
3786 RHSVal = getValue(RHS);
3787 BaseOps.clear();
3788 }
3789
3790 if (IsUnaryAbs) {
3791 OpCode = Opc;
3792 LHSVal = getValue(LHS);
3793 BaseOps.clear();
3794 }
3795 }
3796
3797 if (IsUnaryAbs) {
3798 for (unsigned i = 0; i != NumValues; ++i) {
3799 SDLoc dl = getCurSDLoc();
3800 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3801 Values[i] =
3802 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3803 if (Negate)
3804 Values[i] = DAG.getNegative(Values[i], dl, VT);
3805 }
3806 } else {
3807 for (unsigned i = 0; i != NumValues; ++i) {
3808 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3809 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3810 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3811 Values[i] = DAG.getNode(
3812 OpCode, getCurSDLoc(),
3813 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3814 }
3815 }
3816
3818 DAG.getVTList(ValueVTs), Values));
3819}
3820
3821void SelectionDAGBuilder::visitTrunc(const User &I) {
3822 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3823 SDValue N = getValue(I.getOperand(0));
3825 I.getType());
3827}
3828
3829void SelectionDAGBuilder::visitZExt(const User &I) {
3830 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3831 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3832 SDValue N = getValue(I.getOperand(0));
3833 auto &TLI = DAG.getTargetLoweringInfo();
3834 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3835
3837 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3838 Flags.setNonNeg(PNI->hasNonNeg());
3839
3840 // Eagerly use nonneg information to canonicalize towards sign_extend if
3841 // that is the target's preference.
3842 // TODO: Let the target do this later.
3843 if (Flags.hasNonNeg() &&
3844 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
3846 return;
3847 }
3848
3849 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
3850}
3851
3852void SelectionDAGBuilder::visitSExt(const User &I) {
3853 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3854 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3855 SDValue N = getValue(I.getOperand(0));
3857 I.getType());
3859}
3860
3861void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3862 // FPTrunc is never a no-op cast, no need to check
3863 SDValue N = getValue(I.getOperand(0));
3864 SDLoc dl = getCurSDLoc();
3866 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3867 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3869 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3870}
3871
3872void SelectionDAGBuilder::visitFPExt(const User &I) {
3873 // FPExt is never a no-op cast, no need to check
3874 SDValue N = getValue(I.getOperand(0));
3876 I.getType());
3878}
3879
3880void SelectionDAGBuilder::visitFPToUI(const User &I) {
3881 // FPToUI is never a no-op cast, no need to check
3882 SDValue N = getValue(I.getOperand(0));
3884 I.getType());
3886}
3887
3888void SelectionDAGBuilder::visitFPToSI(const User &I) {
3889 // FPToSI is never a no-op cast, no need to check
3890 SDValue N = getValue(I.getOperand(0));
3892 I.getType());
3894}
3895
3896void SelectionDAGBuilder::visitUIToFP(const User &I) {
3897 // UIToFP is never a no-op cast, no need to check
3898 SDValue N = getValue(I.getOperand(0));
3900 I.getType());
3902 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3903 Flags.setNonNeg(PNI->hasNonNeg());
3904
3905 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
3906}
3907
3908void SelectionDAGBuilder::visitSIToFP(const User &I) {
3909 // SIToFP is never a no-op cast, no need to check
3910 SDValue N = getValue(I.getOperand(0));
3912 I.getType());
3914}
3915
3916void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3917 // What to do depends on the size of the integer and the size of the pointer.
3918 // We can either truncate, zero extend, or no-op, accordingly.
3919 SDValue N = getValue(I.getOperand(0));
3920 auto &TLI = DAG.getTargetLoweringInfo();
3922 I.getType());
3923 EVT PtrMemVT =
3924 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3925 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3926 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3927 setValue(&I, N);
3928}
3929
3930void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3931 // What to do depends on the size of the integer and the size of the pointer.
3932 // We can either truncate, zero extend, or no-op, accordingly.
3933 SDValue N = getValue(I.getOperand(0));
3934 auto &TLI = DAG.getTargetLoweringInfo();
3935 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3936 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3937 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3938 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3939 setValue(&I, N);
3940}
3941
3942void SelectionDAGBuilder::visitBitCast(const User &I) {
3943 SDValue N = getValue(I.getOperand(0));
3944 SDLoc dl = getCurSDLoc();
3946 I.getType());
3947
3948 // BitCast assures us that source and destination are the same size so this is
3949 // either a BITCAST or a no-op.
3950 if (DestVT != N.getValueType())
3952 DestVT, N)); // convert types.
3953 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3954 // might fold any kind of constant expression to an integer constant and that
3955 // is not what we are looking for. Only recognize a bitcast of a genuine
3956 // constant integer as an opaque constant.
3957 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3958 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3959 /*isOpaque*/true));
3960 else
3961 setValue(&I, N); // noop cast.
3962}
3963
3964void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3966 const Value *SV = I.getOperand(0);
3967 SDValue N = getValue(SV);
3968 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3969
3970 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3971 unsigned DestAS = I.getType()->getPointerAddressSpace();
3972
3973 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3974 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3975
3976 setValue(&I, N);
3977}
3978
3979void SelectionDAGBuilder::visitInsertElement(const User &I) {
3981 SDValue InVec = getValue(I.getOperand(0));
3982 SDValue InVal = getValue(I.getOperand(1));
3983 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3986 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3987 InVec, InVal, InIdx));
3988}
3989
3990void SelectionDAGBuilder::visitExtractElement(const User &I) {
3992 SDValue InVec = getValue(I.getOperand(0));
3993 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3996 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3997 InVec, InIdx));
3998}
3999
4000void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4001 SDValue Src1 = getValue(I.getOperand(0));
4002 SDValue Src2 = getValue(I.getOperand(1));
4004 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
4005 Mask = SVI->getShuffleMask();
4006 else
4007 Mask = cast<ConstantExpr>(I).getShuffleMask();
4008 SDLoc DL = getCurSDLoc();
4010 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4011 EVT SrcVT = Src1.getValueType();
4012
4013 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
4014 VT.isScalableVector()) {
4015 // Canonical splat form of first element of first input vector.
4016 SDValue FirstElt =
4019 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4020 return;
4021 }
4022
4023 // For now, we only handle splats for scalable vectors.
4024 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4025 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4026 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4027
4028 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4029 unsigned MaskNumElts = Mask.size();
4030
4031 if (SrcNumElts == MaskNumElts) {
4032 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4033 return;
4034 }
4035
4036 // Normalize the shuffle vector since mask and vector length don't match.
4037 if (SrcNumElts < MaskNumElts) {
4038 // Mask is longer than the source vectors. We can use concatenate vector to
4039 // make the mask and vectors lengths match.
4040
4041 if (MaskNumElts % SrcNumElts == 0) {
4042 // Mask length is a multiple of the source vector length.
4043 // Check if the shuffle is some kind of concatenation of the input
4044 // vectors.
4045 unsigned NumConcat = MaskNumElts / SrcNumElts;
4046 bool IsConcat = true;
4047 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4048 for (unsigned i = 0; i != MaskNumElts; ++i) {
4049 int Idx = Mask[i];
4050 if (Idx < 0)
4051 continue;
4052 // Ensure the indices in each SrcVT sized piece are sequential and that
4053 // the same source is used for the whole piece.
4054 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4055 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4056 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4057 IsConcat = false;
4058 break;
4059 }
4060 // Remember which source this index came from.
4061 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4062 }
4063
4064 // The shuffle is concatenating multiple vectors together. Just emit
4065 // a CONCAT_VECTORS operation.
4066 if (IsConcat) {
4067 SmallVector<SDValue, 8> ConcatOps;
4068 for (auto Src : ConcatSrcs) {
4069 if (Src < 0)
4070 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4071 else if (Src == 0)
4072 ConcatOps.push_back(Src1);
4073 else
4074 ConcatOps.push_back(Src2);
4075 }
4076 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4077 return;
4078 }
4079 }
4080
4081 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4082 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4083 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4084 PaddedMaskNumElts);
4085
4086 // Pad both vectors with undefs to make them the same length as the mask.
4087 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4088
4089 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4090 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4091 MOps1[0] = Src1;
4092 MOps2[0] = Src2;
4093
4094 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4095 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4096
4097 // Readjust mask for new input vector length.
4098 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4099 for (unsigned i = 0; i != MaskNumElts; ++i) {
4100 int Idx = Mask[i];
4101 if (Idx >= (int)SrcNumElts)
4102 Idx -= SrcNumElts - PaddedMaskNumElts;
4103 MappedOps[i] = Idx;
4104 }
4105
4106 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4107
4108 // If the concatenated vector was padded, extract a subvector with the
4109 // correct number of elements.
4110 if (MaskNumElts != PaddedMaskNumElts)
4113
4114 setValue(&I, Result);
4115 return;
4116 }
4117
4118 if (SrcNumElts > MaskNumElts) {
4119 // Analyze the access pattern of the vector to see if we can extract
4120 // two subvectors and do the shuffle.
4121 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
4122 bool CanExtract = true;
4123 for (int Idx : Mask) {
4124 unsigned Input = 0;
4125 if (Idx < 0)
4126 continue;
4127
4128 if (Idx >= (int)SrcNumElts) {
4129 Input = 1;
4130 Idx -= SrcNumElts;
4131 }
4132
4133 // If all the indices come from the same MaskNumElts sized portion of
4134 // the sources we can use extract. Also make sure the extract wouldn't
4135 // extract past the end of the source.
4136 int NewStartIdx = alignDown(Idx, MaskNumElts);
4137 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4138 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4139 CanExtract = false;
4140 // Make sure we always update StartIdx as we use it to track if all
4141 // elements are undef.
4142 StartIdx[Input] = NewStartIdx;
4143 }
4144
4145 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4146 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4147 return;
4148 }
4149 if (CanExtract) {
4150 // Extract appropriate subvector and generate a vector shuffle
4151 for (unsigned Input = 0; Input < 2; ++Input) {
4152 SDValue &Src = Input == 0 ? Src1 : Src2;
4153 if (StartIdx[Input] < 0)
4154 Src = DAG.getUNDEF(VT);
4155 else {
4156 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4157 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4158 }
4159 }
4160
4161 // Calculate new mask.
4162 SmallVector<int, 8> MappedOps(Mask);
4163 for (int &Idx : MappedOps) {
4164 if (Idx >= (int)SrcNumElts)
4165 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4166 else if (Idx >= 0)
4167 Idx -= StartIdx[0];
4168 }
4169
4170 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4171 return;
4172 }
4173 }
4174
4175 // We can't use either concat vectors or extract subvectors so fall back to
4176 // replacing the shuffle with extract and build vector.
4177 // to insert and build vector.
4178 EVT EltVT = VT.getVectorElementType();
4180 for (int Idx : Mask) {
4181 SDValue Res;
4182
4183 if (Idx < 0) {
4184 Res = DAG.getUNDEF(EltVT);
4185 } else {
4186 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4187 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4188
4189 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4191 }
4192
4193 Ops.push_back(Res);
4194 }
4195
4196 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4197}
4198
4199void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4200 ArrayRef<unsigned> Indices = I.getIndices();
4201 const Value *Op0 = I.getOperand(0);
4202 const Value *Op1 = I.getOperand(1);
4203 Type *AggTy = I.getType();
4204 Type *ValTy = Op1->getType();
4205 bool IntoUndef = isa<UndefValue>(Op0);
4206 bool FromUndef = isa<UndefValue>(Op1);
4207
4208 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4209
4211 SmallVector<EVT, 4> AggValueVTs;
4212 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4213 SmallVector<EVT, 4> ValValueVTs;
4214 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4215
4216 unsigned NumAggValues = AggValueVTs.size();
4217 unsigned NumValValues = ValValueVTs.size();
4218 SmallVector<SDValue, 4> Values(NumAggValues);
4219
4220 // Ignore an insertvalue that produces an empty object
4221 if (!NumAggValues) {
4222 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4223 return;
4224 }
4225
4226 SDValue Agg = getValue(Op0);
4227 unsigned i = 0;
4228 // Copy the beginning value(s) from the original aggregate.
4229 for (; i != LinearIndex; ++i)
4230 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4231 SDValue(Agg.getNode(), Agg.getResNo() + i);
4232 // Copy values from the inserted value(s).
4233 if (NumValValues) {
4234 SDValue Val = getValue(Op1);
4235 for (; i != LinearIndex + NumValValues; ++i)
4236 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4237 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4238 }
4239 // Copy remaining value(s) from the original aggregate.
4240 for (; i != NumAggValues; ++i)
4241 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4242 SDValue(Agg.getNode(), Agg.getResNo() + i);
4243
4245 DAG.getVTList(AggValueVTs), Values));
4246}
4247
4248void SelectionDAGB