LLVM 18.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
57#include "llvm/IR/Argument.h"
58#include "llvm/IR/Attributes.h"
59#include "llvm/IR/BasicBlock.h"
60#include "llvm/IR/CFG.h"
61#include "llvm/IR/CallingConv.h"
62#include "llvm/IR/Constant.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/DataLayout.h"
66#include "llvm/IR/DebugInfo.h"
71#include "llvm/IR/Function.h"
73#include "llvm/IR/InlineAsm.h"
74#include "llvm/IR/InstrTypes.h"
77#include "llvm/IR/Intrinsics.h"
78#include "llvm/IR/IntrinsicsAArch64.h"
79#include "llvm/IR/IntrinsicsWebAssembly.h"
80#include "llvm/IR/LLVMContext.h"
81#include "llvm/IR/Metadata.h"
82#include "llvm/IR/Module.h"
83#include "llvm/IR/Operator.h"
85#include "llvm/IR/Statepoint.h"
86#include "llvm/IR/Type.h"
87#include "llvm/IR/User.h"
88#include "llvm/IR/Value.h"
89#include "llvm/MC/MCContext.h"
94#include "llvm/Support/Debug.h"
102#include <cstddef>
103#include <iterator>
104#include <limits>
105#include <optional>
106#include <tuple>
107
108using namespace llvm;
109using namespace PatternMatch;
110using namespace SwitchCG;
111
112#define DEBUG_TYPE "isel"
113
114/// LimitFloatPrecision - Generate low-precision inline sequences for
115/// some float libcalls (6, 8 or 12 bits).
116static unsigned LimitFloatPrecision;
117
118static cl::opt<bool>
119 InsertAssertAlign("insert-assert-align", cl::init(true),
120 cl::desc("Insert the experimental `assertalign` node."),
122
124 LimitFPPrecision("limit-float-precision",
125 cl::desc("Generate low-precision inline sequences "
126 "for some float libcalls"),
128 cl::init(0));
129
131 "switch-peel-threshold", cl::Hidden, cl::init(66),
132 cl::desc("Set the case probability threshold for peeling the case from a "
133 "switch statement. A value greater than 100 will void this "
134 "optimization"));
135
136// Limit the width of DAG chains. This is important in general to prevent
137// DAG-based analysis from blowing up. For example, alias analysis and
138// load clustering may not complete in reasonable time. It is difficult to
139// recognize and avoid this situation within each individual analysis, and
140// future analyses are likely to have the same behavior. Limiting DAG width is
141// the safe approach and will be especially important with global DAGs.
142//
143// MaxParallelChains default is arbitrarily high to avoid affecting
144// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
145// sequence over this should have been converted to llvm.memcpy by the
146// frontend. It is easy to induce this behavior with .ll code such as:
147// %buffer = alloca [4096 x i8]
148// %data = load [4096 x i8]* %argPtr
149// store [4096 x i8] %data, [4096 x i8]* %buffer
150static const unsigned MaxParallelChains = 64;
151
153 const SDValue *Parts, unsigned NumParts,
154 MVT PartVT, EVT ValueVT, const Value *V,
155 std::optional<CallingConv::ID> CC);
156
157/// getCopyFromParts - Create a value that contains the specified legal parts
158/// combined into the value they represent. If the parts combine to a type
159/// larger than ValueVT then AssertOp can be used to specify whether the extra
160/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
161/// (ISD::AssertSext).
162static SDValue
163getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
164 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
165 std::optional<CallingConv::ID> CC = std::nullopt,
166 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
167 // Let the target assemble the parts if it wants to
168 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
169 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
170 PartVT, ValueVT, CC))
171 return Val;
172
173 if (ValueVT.isVector())
174 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
175 CC);
176
177 assert(NumParts > 0 && "No parts to assemble!");
178 SDValue Val = Parts[0];
179
180 if (NumParts > 1) {
181 // Assemble the value from multiple parts.
182 if (ValueVT.isInteger()) {
183 unsigned PartBits = PartVT.getSizeInBits();
184 unsigned ValueBits = ValueVT.getSizeInBits();
185
186 // Assemble the power of 2 part.
187 unsigned RoundParts = llvm::bit_floor(NumParts);
188 unsigned RoundBits = PartBits * RoundParts;
189 EVT RoundVT = RoundBits == ValueBits ?
190 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
191 SDValue Lo, Hi;
192
193 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
194
195 if (RoundParts > 2) {
196 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
197 PartVT, HalfVT, V);
198 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
199 RoundParts / 2, PartVT, HalfVT, V);
200 } else {
201 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
202 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
203 }
204
205 if (DAG.getDataLayout().isBigEndian())
206 std::swap(Lo, Hi);
207
208 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
209
210 if (RoundParts < NumParts) {
211 // Assemble the trailing non-power-of-2 part.
212 unsigned OddParts = NumParts - RoundParts;
213 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
214 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
215 OddVT, V, CC);
216
217 // Combine the round and odd parts.
218 Lo = Val;
219 if (DAG.getDataLayout().isBigEndian())
220 std::swap(Lo, Hi);
221 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
222 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
223 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
224 DAG.getConstant(Lo.getValueSizeInBits(), DL,
226 TotalVT, DAG.getDataLayout())));
227 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
228 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
229 }
230 } else if (PartVT.isFloatingPoint()) {
231 // FP split into multiple FP parts (for ppcf128)
232 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
233 "Unexpected split");
234 SDValue Lo, Hi;
235 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
236 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
237 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
238 std::swap(Lo, Hi);
239 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
240 } else {
241 // FP split into integer parts (soft fp)
242 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
243 !PartVT.isVector() && "Unexpected split");
244 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
245 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
246 }
247 }
248
249 // There is now one part, held in Val. Correct it to match ValueVT.
250 // PartEVT is the type of the register class that holds the value.
251 // ValueVT is the type of the inline asm operation.
252 EVT PartEVT = Val.getValueType();
253
254 if (PartEVT == ValueVT)
255 return Val;
256
257 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
258 ValueVT.bitsLT(PartEVT)) {
259 // For an FP value in an integer part, we need to truncate to the right
260 // width first.
261 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
262 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
263 }
264
265 // Handle types that have the same size.
266 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
267 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
268
269 // Handle types with different sizes.
270 if (PartEVT.isInteger() && ValueVT.isInteger()) {
271 if (ValueVT.bitsLT(PartEVT)) {
272 // For a truncate, see if we have any information to
273 // indicate whether the truncated bits will always be
274 // zero or sign-extension.
275 if (AssertOp)
276 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
277 DAG.getValueType(ValueVT));
278 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
279 }
280 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
281 }
282
283 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
284 // FP_ROUND's are always exact here.
285 if (ValueVT.bitsLT(Val.getValueType()))
286 return DAG.getNode(
287 ISD::FP_ROUND, DL, ValueVT, Val,
288 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
289
290 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
291 }
292
293 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
294 // then truncating.
295 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
296 ValueVT.bitsLT(PartEVT)) {
297 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
298 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
299 }
300
301 report_fatal_error("Unknown mismatch in getCopyFromParts!");
302}
303
305 const Twine &ErrMsg) {
306 const Instruction *I = dyn_cast_or_null<Instruction>(V);
307 if (!V)
308 return Ctx.emitError(ErrMsg);
309
310 const char *AsmError = ", possible invalid constraint for vector type";
311 if (const CallInst *CI = dyn_cast<CallInst>(I))
312 if (CI->isInlineAsm())
313 return Ctx.emitError(I, ErrMsg + AsmError);
314
315 return Ctx.emitError(I, ErrMsg);
316}
317
318/// getCopyFromPartsVector - Create a value that contains the specified legal
319/// parts combined into the value they represent. If the parts combine to a
320/// type larger than ValueVT then AssertOp can be used to specify whether the
321/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
322/// ValueVT (ISD::AssertSext).
324 const SDValue *Parts, unsigned NumParts,
325 MVT PartVT, EVT ValueVT, const Value *V,
326 std::optional<CallingConv::ID> CallConv) {
327 assert(ValueVT.isVector() && "Not a vector value");
328 assert(NumParts > 0 && "No parts to assemble!");
329 const bool IsABIRegCopy = CallConv.has_value();
330
331 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
332 SDValue Val = Parts[0];
333
334 // Handle a multi-element vector.
335 if (NumParts > 1) {
336 EVT IntermediateVT;
337 MVT RegisterVT;
338 unsigned NumIntermediates;
339 unsigned NumRegs;
340
341 if (IsABIRegCopy) {
343 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
344 NumIntermediates, RegisterVT);
345 } else {
346 NumRegs =
347 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
348 NumIntermediates, RegisterVT);
349 }
350
351 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
352 NumParts = NumRegs; // Silence a compiler warning.
353 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
354 assert(RegisterVT.getSizeInBits() ==
355 Parts[0].getSimpleValueType().getSizeInBits() &&
356 "Part type sizes don't match!");
357
358 // Assemble the parts into intermediate operands.
359 SmallVector<SDValue, 8> Ops(NumIntermediates);
360 if (NumIntermediates == NumParts) {
361 // If the register was not expanded, truncate or copy the value,
362 // as appropriate.
363 for (unsigned i = 0; i != NumParts; ++i)
364 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
365 PartVT, IntermediateVT, V, CallConv);
366 } else if (NumParts > 0) {
367 // If the intermediate type was expanded, build the intermediate
368 // operands from the parts.
369 assert(NumParts % NumIntermediates == 0 &&
370 "Must expand into a divisible number of parts!");
371 unsigned Factor = NumParts / NumIntermediates;
372 for (unsigned i = 0; i != NumIntermediates; ++i)
373 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
374 PartVT, IntermediateVT, V, CallConv);
375 }
376
377 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
378 // intermediate operands.
379 EVT BuiltVectorTy =
380 IntermediateVT.isVector()
382 *DAG.getContext(), IntermediateVT.getScalarType(),
383 IntermediateVT.getVectorElementCount() * NumParts)
385 IntermediateVT.getScalarType(),
386 NumIntermediates);
387 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
389 DL, BuiltVectorTy, Ops);
390 }
391
392 // There is now one part, held in Val. Correct it to match ValueVT.
393 EVT PartEVT = Val.getValueType();
394
395 if (PartEVT == ValueVT)
396 return Val;
397
398 if (PartEVT.isVector()) {
399 // Vector/Vector bitcast.
400 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
401 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
402
403 // If the parts vector has more elements than the value vector, then we
404 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
405 // Extract the elements we want.
406 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
409 (PartEVT.getVectorElementCount().isScalable() ==
410 ValueVT.getVectorElementCount().isScalable()) &&
411 "Cannot narrow, it would be a lossy transformation");
412 PartEVT =
414 ValueVT.getVectorElementCount());
415 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
416 DAG.getVectorIdxConstant(0, DL));
417 if (PartEVT == ValueVT)
418 return Val;
419 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
420 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
421
422 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
423 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
424 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
425 }
426
427 // Promoted vector extract
428 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
429 }
430
431 // Trivial bitcast if the types are the same size and the destination
432 // vector type is legal.
433 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
434 TLI.isTypeLegal(ValueVT))
435 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
436
437 if (ValueVT.getVectorNumElements() != 1) {
438 // Certain ABIs require that vectors are passed as integers. For vectors
439 // are the same size, this is an obvious bitcast.
440 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
441 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
442 } else if (ValueVT.bitsLT(PartEVT)) {
443 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
444 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
445 // Drop the extra bits.
446 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
447 return DAG.getBitcast(ValueVT, Val);
448 }
449
451 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
452 return DAG.getUNDEF(ValueVT);
453 }
454
455 // Handle cases such as i8 -> <1 x i1>
456 EVT ValueSVT = ValueVT.getVectorElementType();
457 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
458 unsigned ValueSize = ValueSVT.getSizeInBits();
459 if (ValueSize == PartEVT.getSizeInBits()) {
460 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
461 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
462 // It's possible a scalar floating point type gets softened to integer and
463 // then promoted to a larger integer. If PartEVT is the larger integer
464 // we need to truncate it and then bitcast to the FP type.
465 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
466 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
467 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
468 Val = DAG.getBitcast(ValueSVT, Val);
469 } else {
470 Val = ValueVT.isFloatingPoint()
471 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
472 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
473 }
474 }
475
476 return DAG.getBuildVector(ValueVT, DL, Val);
477}
478
479static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
480 SDValue Val, SDValue *Parts, unsigned NumParts,
481 MVT PartVT, const Value *V,
482 std::optional<CallingConv::ID> CallConv);
483
484/// getCopyToParts - Create a series of nodes that contain the specified value
485/// split into legal parts. If the parts contain more bits than Val, then, for
486/// integers, ExtendKind can be used to specify how to generate the extra bits.
487static void
489 unsigned NumParts, MVT PartVT, const Value *V,
490 std::optional<CallingConv::ID> CallConv = std::nullopt,
491 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
492 // Let the target split the parts if it wants to
493 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
494 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
495 CallConv))
496 return;
497 EVT ValueVT = Val.getValueType();
498
499 // Handle the vector case separately.
500 if (ValueVT.isVector())
501 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
502 CallConv);
503
504 unsigned OrigNumParts = NumParts;
506 "Copying to an illegal type!");
507
508 if (NumParts == 0)
509 return;
510
511 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
512 EVT PartEVT = PartVT;
513 if (PartEVT == ValueVT) {
514 assert(NumParts == 1 && "No-op copy with multiple parts!");
515 Parts[0] = Val;
516 return;
517 }
518
519 unsigned PartBits = PartVT.getSizeInBits();
520 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
521 // If the parts cover more bits than the value has, promote the value.
522 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
523 assert(NumParts == 1 && "Do not know what to promote to!");
524 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
525 } else {
526 if (ValueVT.isFloatingPoint()) {
527 // FP values need to be bitcast, then extended if they are being put
528 // into a larger container.
529 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
530 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
531 }
532 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
533 ValueVT.isInteger() &&
534 "Unknown mismatch!");
535 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
536 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
537 if (PartVT == MVT::x86mmx)
538 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
539 }
540 } else if (PartBits == ValueVT.getSizeInBits()) {
541 // Different types of the same size.
542 assert(NumParts == 1 && PartEVT != ValueVT);
543 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
544 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
545 // If the parts cover less bits than value has, truncate the value.
546 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
547 ValueVT.isInteger() &&
548 "Unknown mismatch!");
549 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
550 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
551 if (PartVT == MVT::x86mmx)
552 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
553 }
554
555 // The value may have changed - recompute ValueVT.
556 ValueVT = Val.getValueType();
557 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
558 "Failed to tile the value with PartVT!");
559
560 if (NumParts == 1) {
561 if (PartEVT != ValueVT) {
563 "scalar-to-vector conversion failed");
564 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
565 }
566
567 Parts[0] = Val;
568 return;
569 }
570
571 // Expand the value into multiple parts.
572 if (NumParts & (NumParts - 1)) {
573 // The number of parts is not a power of 2. Split off and copy the tail.
574 assert(PartVT.isInteger() && ValueVT.isInteger() &&
575 "Do not know what to expand to!");
576 unsigned RoundParts = llvm::bit_floor(NumParts);
577 unsigned RoundBits = RoundParts * PartBits;
578 unsigned OddParts = NumParts - RoundParts;
579 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
580 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
581
582 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
583 CallConv);
584
585 if (DAG.getDataLayout().isBigEndian())
586 // The odd parts were reversed by getCopyToParts - unreverse them.
587 std::reverse(Parts + RoundParts, Parts + NumParts);
588
589 NumParts = RoundParts;
590 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
591 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
592 }
593
594 // The number of parts is a power of 2. Repeatedly bisect the value using
595 // EXTRACT_ELEMENT.
596 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
598 ValueVT.getSizeInBits()),
599 Val);
600
601 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
602 for (unsigned i = 0; i < NumParts; i += StepSize) {
603 unsigned ThisBits = StepSize * PartBits / 2;
604 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
605 SDValue &Part0 = Parts[i];
606 SDValue &Part1 = Parts[i+StepSize/2];
607
608 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
609 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
610 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
611 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
612
613 if (ThisBits == PartBits && ThisVT != PartVT) {
614 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
615 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
616 }
617 }
618 }
619
620 if (DAG.getDataLayout().isBigEndian())
621 std::reverse(Parts, Parts + OrigNumParts);
622}
623
625 const SDLoc &DL, EVT PartVT) {
626 if (!PartVT.isVector())
627 return SDValue();
628
629 EVT ValueVT = Val.getValueType();
630 EVT PartEVT = PartVT.getVectorElementType();
631 EVT ValueEVT = ValueVT.getVectorElementType();
632 ElementCount PartNumElts = PartVT.getVectorElementCount();
633 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
634
635 // We only support widening vectors with equivalent element types and
636 // fixed/scalable properties. If a target needs to widen a fixed-length type
637 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
638 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
639 PartNumElts.isScalable() != ValueNumElts.isScalable())
640 return SDValue();
641
642 // Have a try for bf16 because some targets share its ABI with fp16.
643 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
645 "Cannot widen to illegal type");
646 Val = DAG.getNode(ISD::BITCAST, DL,
647 ValueVT.changeVectorElementType(MVT::f16), Val);
648 } else if (PartEVT != ValueEVT) {
649 return SDValue();
650 }
651
652 // Widening a scalable vector to another scalable vector is done by inserting
653 // the vector into a larger undef one.
654 if (PartNumElts.isScalable())
655 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
656 Val, DAG.getVectorIdxConstant(0, DL));
657
658 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
659 // undef elements.
661 DAG.ExtractVectorElements(Val, Ops);
662 SDValue EltUndef = DAG.getUNDEF(PartEVT);
663 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
664
665 // FIXME: Use CONCAT for 2x -> 4x.
666 return DAG.getBuildVector(PartVT, DL, Ops);
667}
668
669/// getCopyToPartsVector - Create a series of nodes that contain the specified
670/// value split into legal parts.
671static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
672 SDValue Val, SDValue *Parts, unsigned NumParts,
673 MVT PartVT, const Value *V,
674 std::optional<CallingConv::ID> CallConv) {
675 EVT ValueVT = Val.getValueType();
676 assert(ValueVT.isVector() && "Not a vector");
677 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
678 const bool IsABIRegCopy = CallConv.has_value();
679
680 if (NumParts == 1) {
681 EVT PartEVT = PartVT;
682 if (PartEVT == ValueVT) {
683 // Nothing to do.
684 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
685 // Bitconvert vector->vector case.
686 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
687 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
688 Val = Widened;
689 } else if (PartVT.isVector() &&
691 ValueVT.getVectorElementType()) &&
692 PartEVT.getVectorElementCount() ==
693 ValueVT.getVectorElementCount()) {
694
695 // Promoted vector extract
696 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
697 } else if (PartEVT.isVector() &&
698 PartEVT.getVectorElementType() !=
699 ValueVT.getVectorElementType() &&
700 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
701 TargetLowering::TypeWidenVector) {
702 // Combination of widening and promotion.
703 EVT WidenVT =
705 PartVT.getVectorElementCount());
706 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
707 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
708 } else {
709 // Don't extract an integer from a float vector. This can happen if the
710 // FP type gets softened to integer and then promoted. The promotion
711 // prevents it from being picked up by the earlier bitcast case.
712 if (ValueVT.getVectorElementCount().isScalar() &&
713 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
714 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
715 DAG.getVectorIdxConstant(0, DL));
716 } else {
717 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
718 assert(PartVT.getFixedSizeInBits() > ValueSize &&
719 "lossy conversion of vector to scalar type");
720 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
721 Val = DAG.getBitcast(IntermediateType, Val);
722 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
723 }
724 }
725
726 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
727 Parts[0] = Val;
728 return;
729 }
730
731 // Handle a multi-element vector.
732 EVT IntermediateVT;
733 MVT RegisterVT;
734 unsigned NumIntermediates;
735 unsigned NumRegs;
736 if (IsABIRegCopy) {
738 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
739 RegisterVT);
740 } else {
741 NumRegs =
742 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
743 NumIntermediates, RegisterVT);
744 }
745
746 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
747 NumParts = NumRegs; // Silence a compiler warning.
748 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
749
750 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
751 "Mixing scalable and fixed vectors when copying in parts");
752
753 std::optional<ElementCount> DestEltCnt;
754
755 if (IntermediateVT.isVector())
756 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
757 else
758 DestEltCnt = ElementCount::getFixed(NumIntermediates);
759
760 EVT BuiltVectorTy = EVT::getVectorVT(
761 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
762
763 if (ValueVT == BuiltVectorTy) {
764 // Nothing to do.
765 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
766 // Bitconvert vector->vector case.
767 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
768 } else {
769 if (BuiltVectorTy.getVectorElementType().bitsGT(
770 ValueVT.getVectorElementType())) {
771 // Integer promotion.
772 ValueVT = EVT::getVectorVT(*DAG.getContext(),
773 BuiltVectorTy.getVectorElementType(),
774 ValueVT.getVectorElementCount());
775 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
776 }
777
778 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
779 Val = Widened;
780 }
781 }
782
783 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
784
785 // Split the vector into intermediate operands.
786 SmallVector<SDValue, 8> Ops(NumIntermediates);
787 for (unsigned i = 0; i != NumIntermediates; ++i) {
788 if (IntermediateVT.isVector()) {
789 // This does something sensible for scalable vectors - see the
790 // definition of EXTRACT_SUBVECTOR for further details.
791 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
792 Ops[i] =
793 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
794 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
795 } else {
796 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
797 DAG.getVectorIdxConstant(i, DL));
798 }
799 }
800
801 // Split the intermediate operands into legal parts.
802 if (NumParts == NumIntermediates) {
803 // If the register was not expanded, promote or copy the value,
804 // as appropriate.
805 for (unsigned i = 0; i != NumParts; ++i)
806 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
807 } else if (NumParts > 0) {
808 // If the intermediate type was expanded, split each the value into
809 // legal parts.
810 assert(NumIntermediates != 0 && "division by zero");
811 assert(NumParts % NumIntermediates == 0 &&
812 "Must expand into a divisible number of parts!");
813 unsigned Factor = NumParts / NumIntermediates;
814 for (unsigned i = 0; i != NumIntermediates; ++i)
815 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
816 CallConv);
817 }
818}
819
821 EVT valuevt, std::optional<CallingConv::ID> CC)
822 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
823 RegCount(1, regs.size()), CallConv(CC) {}
824
826 const DataLayout &DL, unsigned Reg, Type *Ty,
827 std::optional<CallingConv::ID> CC) {
828 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
829
830 CallConv = CC;
831
832 for (EVT ValueVT : ValueVTs) {
833 unsigned NumRegs =
836 : TLI.getNumRegisters(Context, ValueVT);
837 MVT RegisterVT =
840 : TLI.getRegisterType(Context, ValueVT);
841 for (unsigned i = 0; i != NumRegs; ++i)
842 Regs.push_back(Reg + i);
843 RegVTs.push_back(RegisterVT);
844 RegCount.push_back(NumRegs);
845 Reg += NumRegs;
846 }
847}
848
850 FunctionLoweringInfo &FuncInfo,
851 const SDLoc &dl, SDValue &Chain,
852 SDValue *Glue, const Value *V) const {
853 // A Value with type {} or [0 x %t] needs no registers.
854 if (ValueVTs.empty())
855 return SDValue();
856
857 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
858
859 // Assemble the legal parts into the final values.
860 SmallVector<SDValue, 4> Values(ValueVTs.size());
862 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
863 // Copy the legal parts from the registers.
864 EVT ValueVT = ValueVTs[Value];
865 unsigned NumRegs = RegCount[Value];
866 MVT RegisterVT = isABIMangled()
868 *DAG.getContext(), *CallConv, RegVTs[Value])
869 : RegVTs[Value];
870
871 Parts.resize(NumRegs);
872 for (unsigned i = 0; i != NumRegs; ++i) {
873 SDValue P;
874 if (!Glue) {
875 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
876 } else {
877 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
878 *Glue = P.getValue(2);
879 }
880
881 Chain = P.getValue(1);
882 Parts[i] = P;
883
884 // If the source register was virtual and if we know something about it,
885 // add an assert node.
886 if (!Register::isVirtualRegister(Regs[Part + i]) ||
887 !RegisterVT.isInteger())
888 continue;
889
891 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
892 if (!LOI)
893 continue;
894
895 unsigned RegSize = RegisterVT.getScalarSizeInBits();
896 unsigned NumSignBits = LOI->NumSignBits;
897 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
898
899 if (NumZeroBits == RegSize) {
900 // The current value is a zero.
901 // Explicitly express that as it would be easier for
902 // optimizations to kick in.
903 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
904 continue;
905 }
906
907 // FIXME: We capture more information than the dag can represent. For
908 // now, just use the tightest assertzext/assertsext possible.
909 bool isSExt;
910 EVT FromVT(MVT::Other);
911 if (NumZeroBits) {
912 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
913 isSExt = false;
914 } else if (NumSignBits > 1) {
915 FromVT =
916 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
917 isSExt = true;
918 } else {
919 continue;
920 }
921 // Add an assertion node.
922 assert(FromVT != MVT::Other);
923 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
924 RegisterVT, P, DAG.getValueType(FromVT));
925 }
926
927 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
928 RegisterVT, ValueVT, V, CallConv);
929 Part += NumRegs;
930 Parts.clear();
931 }
932
933 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
934}
935
937 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
938 const Value *V,
939 ISD::NodeType PreferredExtendType) const {
940 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
941 ISD::NodeType ExtendKind = PreferredExtendType;
942
943 // Get the list of the values's legal parts.
944 unsigned NumRegs = Regs.size();
945 SmallVector<SDValue, 8> Parts(NumRegs);
946 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
947 unsigned NumParts = RegCount[Value];
948
949 MVT RegisterVT = isABIMangled()
951 *DAG.getContext(), *CallConv, RegVTs[Value])
952 : RegVTs[Value];
953
954 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
955 ExtendKind = ISD::ZERO_EXTEND;
956
957 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
958 NumParts, RegisterVT, V, CallConv, ExtendKind);
959 Part += NumParts;
960 }
961
962 // Copy the parts into the registers.
963 SmallVector<SDValue, 8> Chains(NumRegs);
964 for (unsigned i = 0; i != NumRegs; ++i) {
965 SDValue Part;
966 if (!Glue) {
967 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
968 } else {
969 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
970 *Glue = Part.getValue(1);
971 }
972
973 Chains[i] = Part.getValue(0);
974 }
975
976 if (NumRegs == 1 || Glue)
977 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
978 // flagged to it. That is the CopyToReg nodes and the user are considered
979 // a single scheduling unit. If we create a TokenFactor and return it as
980 // chain, then the TokenFactor is both a predecessor (operand) of the
981 // user as well as a successor (the TF operands are flagged to the user).
982 // c1, f1 = CopyToReg
983 // c2, f2 = CopyToReg
984 // c3 = TokenFactor c1, c2
985 // ...
986 // = op c3, ..., f2
987 Chain = Chains[NumRegs-1];
988 else
989 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
990}
991
993 unsigned MatchingIdx, const SDLoc &dl,
994 SelectionDAG &DAG,
995 std::vector<SDValue> &Ops) const {
996 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
997
998 InlineAsm::Flag Flag(Code, Regs.size());
999 if (HasMatching)
1000 Flag.setMatchingOp(MatchingIdx);
1001 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
1002 // Put the register class of the virtual registers in the flag word. That
1003 // way, later passes can recompute register class constraints for inline
1004 // assembly as well as normal instructions.
1005 // Don't do this for tied operands that can use the regclass information
1006 // from the def.
1008 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1009 Flag.setRegClass(RC->getID());
1010 }
1011
1012 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1013 Ops.push_back(Res);
1014
1015 if (Code == InlineAsm::Kind::Clobber) {
1016 // Clobbers should always have a 1:1 mapping with registers, and may
1017 // reference registers that have illegal (e.g. vector) types. Hence, we
1018 // shouldn't try to apply any sort of splitting logic to them.
1019 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1020 "No 1:1 mapping from clobbers to regs?");
1022 (void)SP;
1023 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1024 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1025 assert(
1026 (Regs[I] != SP ||
1028 "If we clobbered the stack pointer, MFI should know about it.");
1029 }
1030 return;
1031 }
1032
1033 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1034 MVT RegisterVT = RegVTs[Value];
1035 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1036 RegisterVT);
1037 for (unsigned i = 0; i != NumRegs; ++i) {
1038 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1039 unsigned TheReg = Regs[Reg++];
1040 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1041 }
1042 }
1043}
1044
1048 unsigned I = 0;
1049 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1050 unsigned RegCount = std::get<0>(CountAndVT);
1051 MVT RegisterVT = std::get<1>(CountAndVT);
1052 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1053 for (unsigned E = I + RegCount; I != E; ++I)
1054 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1055 }
1056 return OutVec;
1057}
1058
1060 AssumptionCache *ac,
1061 const TargetLibraryInfo *li) {
1062 AA = aa;
1063 AC = ac;
1064 GFI = gfi;
1065 LibInfo = li;
1067 LPadToCallSiteMap.clear();
1069 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1071}
1072
1074 NodeMap.clear();
1075 UnusedArgNodeMap.clear();
1076 PendingLoads.clear();
1077 PendingExports.clear();
1078 PendingConstrainedFP.clear();
1079 PendingConstrainedFPStrict.clear();
1080 CurInst = nullptr;
1081 HasTailCall = false;
1082 SDNodeOrder = LowestSDNodeOrder;
1084}
1085
1087 DanglingDebugInfoMap.clear();
1088}
1089
1090// Update DAG root to include dependencies on Pending chains.
1091SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1092 SDValue Root = DAG.getRoot();
1093
1094 if (Pending.empty())
1095 return Root;
1096
1097 // Add current root to PendingChains, unless we already indirectly
1098 // depend on it.
1099 if (Root.getOpcode() != ISD::EntryToken) {
1100 unsigned i = 0, e = Pending.size();
1101 for (; i != e; ++i) {
1102 assert(Pending[i].getNode()->getNumOperands() > 1);
1103 if (Pending[i].getNode()->getOperand(0) == Root)
1104 break; // Don't add the root if we already indirectly depend on it.
1105 }
1106
1107 if (i == e)
1108 Pending.push_back(Root);
1109 }
1110
1111 if (Pending.size() == 1)
1112 Root = Pending[0];
1113 else
1114 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1115
1116 DAG.setRoot(Root);
1117 Pending.clear();
1118 return Root;
1119}
1120
1122 return updateRoot(PendingLoads);
1123}
1124
1126 // Chain up all pending constrained intrinsics together with all
1127 // pending loads, by simply appending them to PendingLoads and
1128 // then calling getMemoryRoot().
1129 PendingLoads.reserve(PendingLoads.size() +
1130 PendingConstrainedFP.size() +
1131 PendingConstrainedFPStrict.size());
1132 PendingLoads.append(PendingConstrainedFP.begin(),
1133 PendingConstrainedFP.end());
1134 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1135 PendingConstrainedFPStrict.end());
1136 PendingConstrainedFP.clear();
1137 PendingConstrainedFPStrict.clear();
1138 return getMemoryRoot();
1139}
1140
1142 // We need to emit pending fpexcept.strict constrained intrinsics,
1143 // so append them to the PendingExports list.
1144 PendingExports.append(PendingConstrainedFPStrict.begin(),
1145 PendingConstrainedFPStrict.end());
1146 PendingConstrainedFPStrict.clear();
1147 return updateRoot(PendingExports);
1148}
1149
1151 // Set up outgoing PHI node register values before emitting the terminator.
1152 if (I.isTerminator()) {
1153 HandlePHINodesInSuccessorBlocks(I.getParent());
1154 }
1155
1156 // Add SDDbgValue nodes for any var locs here. Do so before updating
1157 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1158 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1159 // Add SDDbgValue nodes for any var locs here. Do so before updating
1160 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1161 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1162 It != End; ++It) {
1163 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1164 dropDanglingDebugInfo(Var, It->Expr);
1165 if (It->Values.isKillLocation(It->Expr)) {
1166 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1167 continue;
1168 }
1169 SmallVector<Value *> Values(It->Values.location_ops());
1170 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1171 It->Values.hasArgList()))
1172 addDanglingDebugInfo(It, SDNodeOrder);
1173 }
1174 }
1175
1176 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1177 if (!isa<DbgInfoIntrinsic>(I))
1178 ++SDNodeOrder;
1179
1180 CurInst = &I;
1181
1182 // Set inserted listener only if required.
1183 bool NodeInserted = false;
1184 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1185 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1186 if (PCSectionsMD) {
1187 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1188 DAG, [&](SDNode *) { NodeInserted = true; });
1189 }
1190
1191 visit(I.getOpcode(), I);
1192
1193 if (!I.isTerminator() && !HasTailCall &&
1194 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1196
1197 // Handle metadata.
1198 if (PCSectionsMD) {
1199 auto It = NodeMap.find(&I);
1200 if (It != NodeMap.end()) {
1201 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1202 } else if (NodeInserted) {
1203 // This should not happen; if it does, don't let it go unnoticed so we can
1204 // fix it. Relevant visit*() function is probably missing a setValue().
1205 errs() << "warning: loosing !pcsections metadata ["
1206 << I.getModule()->getName() << "]\n";
1207 LLVM_DEBUG(I.dump());
1208 assert(false);
1209 }
1210 }
1211
1212 CurInst = nullptr;
1213}
1214
1215void SelectionDAGBuilder::visitPHI(const PHINode &) {
1216 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1217}
1218
1219void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1220 // Note: this doesn't use InstVisitor, because it has to work with
1221 // ConstantExpr's in addition to instructions.
1222 switch (Opcode) {
1223 default: llvm_unreachable("Unknown instruction type encountered!");
1224 // Build the switch statement using the Instruction.def file.
1225#define HANDLE_INST(NUM, OPCODE, CLASS) \
1226 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1227#include "llvm/IR/Instruction.def"
1228 }
1229}
1230
1232 DILocalVariable *Variable,
1233 DebugLoc DL, unsigned Order,
1234 RawLocationWrapper Values,
1236 if (!Values.hasArgList())
1237 return false;
1238 // For variadic dbg_values we will now insert an undef.
1239 // FIXME: We can potentially recover these!
1241 for (const Value *V : Values.location_ops()) {
1242 auto *Undef = UndefValue::get(V->getType());
1244 }
1245 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1246 /*IsIndirect=*/false, DL, Order,
1247 /*IsVariadic=*/true);
1248 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1249 return true;
1250}
1251
1253 unsigned Order) {
1255 DAG,
1256 const_cast<DILocalVariable *>(DAG.getFunctionVarLocs()
1257 ->getVariable(VarLoc->VariableID)
1258 .getVariable()),
1259 VarLoc->DL, Order, VarLoc->Values, VarLoc->Expr)) {
1260 DanglingDebugInfoMap[VarLoc->Values.getVariableLocationOp(0)].emplace_back(
1261 VarLoc, Order);
1262 }
1263}
1264
1266 unsigned Order) {
1267 // We treat variadic dbg_values differently at this stage.
1269 DAG, DI->getVariable(), DI->getDebugLoc(), Order,
1270 DI->getWrappedLocation(), DI->getExpression())) {
1271 // TODO: Dangling debug info will eventually either be resolved or produce
1272 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1273 // between the original dbg.value location and its resolved DBG_VALUE,
1274 // which we should ideally fill with an extra Undef DBG_VALUE.
1275 assert(DI->getNumVariableLocationOps() == 1 &&
1276 "DbgValueInst without an ArgList should have a single location "
1277 "operand.");
1278 DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, Order);
1279 }
1280}
1281
1283 const DIExpression *Expr) {
1284 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1285 DIVariable *DanglingVariable = DDI.getVariable(DAG.getFunctionVarLocs());
1286 DIExpression *DanglingExpr = DDI.getExpression();
1287 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1288 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << printDDI(DDI)
1289 << "\n");
1290 return true;
1291 }
1292 return false;
1293 };
1294
1295 for (auto &DDIMI : DanglingDebugInfoMap) {
1296 DanglingDebugInfoVector &DDIV = DDIMI.second;
1297
1298 // If debug info is to be dropped, run it through final checks to see
1299 // whether it can be salvaged.
1300 for (auto &DDI : DDIV)
1301 if (isMatchingDbgValue(DDI))
1303
1304 erase_if(DDIV, isMatchingDbgValue);
1305 }
1306}
1307
1308// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1309// generate the debug data structures now that we've seen its definition.
1311 SDValue Val) {
1312 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1313 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1314 return;
1315
1316 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1317 for (auto &DDI : DDIV) {
1318 DebugLoc DL = DDI.getDebugLoc();
1319 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1320 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1321 DILocalVariable *Variable = DDI.getVariable(DAG.getFunctionVarLocs());
1322 DIExpression *Expr = DDI.getExpression();
1324 "Expected inlined-at fields to agree");
1325 SDDbgValue *SDV;
1326 if (Val.getNode()) {
1327 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1328 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1329 // we couldn't resolve it directly when examining the DbgValue intrinsic
1330 // in the first place we should not be more successful here). Unless we
1331 // have some test case that prove this to be correct we should avoid
1332 // calling EmitFuncArgumentDbgValue here.
1333 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1334 FuncArgumentDbgValueKind::Value, Val)) {
1335 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for " << printDDI(DDI)
1336 << "\n");
1337 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1338 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1339 // inserted after the definition of Val when emitting the instructions
1340 // after ISel. An alternative could be to teach
1341 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1342 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1343 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1344 << ValSDNodeOrder << "\n");
1345 SDV = getDbgValue(Val, Variable, Expr, DL,
1346 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1347 DAG.AddDbgValue(SDV, false);
1348 } else
1349 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1350 << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n");
1351 } else {
1352 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(DDI) << "\n");
1353 auto Undef = UndefValue::get(V->getType());
1354 auto SDV =
1355 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1356 DAG.AddDbgValue(SDV, false);
1357 }
1358 }
1359 DDIV.clear();
1360}
1361
1363 // TODO: For the variadic implementation, instead of only checking the fail
1364 // state of `handleDebugValue`, we need know specifically which values were
1365 // invalid, so that we attempt to salvage only those values when processing
1366 // a DIArgList.
1367 Value *V = DDI.getVariableLocationOp(0);
1368 Value *OrigV = V;
1369 DILocalVariable *Var = DDI.getVariable(DAG.getFunctionVarLocs());
1370 DIExpression *Expr = DDI.getExpression();
1371 DebugLoc DL = DDI.getDebugLoc();
1372 unsigned SDOrder = DDI.getSDNodeOrder();
1373
1374 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1375 // that DW_OP_stack_value is desired.
1376 bool StackValue = true;
1377
1378 // Can this Value can be encoded without any further work?
1379 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1380 return;
1381
1382 // Attempt to salvage back through as many instructions as possible. Bail if
1383 // a non-instruction is seen, such as a constant expression or global
1384 // variable. FIXME: Further work could recover those too.
1385 while (isa<Instruction>(V)) {
1386 Instruction &VAsInst = *cast<Instruction>(V);
1387 // Temporary "0", awaiting real implementation.
1389 SmallVector<Value *, 4> AdditionalValues;
1390 V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1391 AdditionalValues);
1392 // If we cannot salvage any further, and haven't yet found a suitable debug
1393 // expression, bail out.
1394 if (!V)
1395 break;
1396
1397 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1398 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1399 // here for variadic dbg_values, remove that condition.
1400 if (!AdditionalValues.empty())
1401 break;
1402
1403 // New value and expr now represent this debuginfo.
1404 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1405
1406 // Some kind of simplification occurred: check whether the operand of the
1407 // salvaged debug expression can be encoded in this DAG.
1408 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1409 LLVM_DEBUG(
1410 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1411 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1412 return;
1413 }
1414 }
1415
1416 // This was the final opportunity to salvage this debug information, and it
1417 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1418 // any earlier variable location.
1419 assert(OrigV && "V shouldn't be null");
1420 auto *Undef = UndefValue::get(OrigV->getType());
1421 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1422 DAG.AddDbgValue(SDV, false);
1423 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << printDDI(DDI)
1424 << "\n");
1425}
1426
1428 DIExpression *Expr,
1429 DebugLoc DbgLoc,
1430 unsigned Order) {
1434 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1435 /*IsVariadic*/ false);
1436}
1437
1439 DILocalVariable *Var,
1440 DIExpression *Expr, DebugLoc DbgLoc,
1441 unsigned Order, bool IsVariadic) {
1442 if (Values.empty())
1443 return true;
1444 SmallVector<SDDbgOperand> LocationOps;
1445 SmallVector<SDNode *> Dependencies;
1446 for (const Value *V : Values) {
1447 // Constant value.
1448 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1449 isa<ConstantPointerNull>(V)) {
1450 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1451 continue;
1452 }
1453
1454 // Look through IntToPtr constants.
1455 if (auto *CE = dyn_cast<ConstantExpr>(V))
1456 if (CE->getOpcode() == Instruction::IntToPtr) {
1457 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1458 continue;
1459 }
1460
1461 // If the Value is a frame index, we can create a FrameIndex debug value
1462 // without relying on the DAG at all.
1463 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1464 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1465 if (SI != FuncInfo.StaticAllocaMap.end()) {
1466 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1467 continue;
1468 }
1469 }
1470
1471 // Do not use getValue() in here; we don't want to generate code at
1472 // this point if it hasn't been done yet.
1473 SDValue N = NodeMap[V];
1474 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1475 N = UnusedArgNodeMap[V];
1476 if (N.getNode()) {
1477 // Only emit func arg dbg value for non-variadic dbg.values for now.
1478 if (!IsVariadic &&
1479 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1480 FuncArgumentDbgValueKind::Value, N))
1481 return true;
1482 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1483 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1484 // describe stack slot locations.
1485 //
1486 // Consider "int x = 0; int *px = &x;". There are two kinds of
1487 // interesting debug values here after optimization:
1488 //
1489 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1490 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1491 //
1492 // Both describe the direct values of their associated variables.
1493 Dependencies.push_back(N.getNode());
1494 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1495 continue;
1496 }
1497 LocationOps.emplace_back(
1498 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1499 continue;
1500 }
1501
1503 // Special rules apply for the first dbg.values of parameter variables in a
1504 // function. Identify them by the fact they reference Argument Values, that
1505 // they're parameters, and they are parameters of the current function. We
1506 // need to let them dangle until they get an SDNode.
1507 bool IsParamOfFunc =
1508 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1509 if (IsParamOfFunc)
1510 return false;
1511
1512 // The value is not used in this block yet (or it would have an SDNode).
1513 // We still want the value to appear for the user if possible -- if it has
1514 // an associated VReg, we can refer to that instead.
1515 auto VMI = FuncInfo.ValueMap.find(V);
1516 if (VMI != FuncInfo.ValueMap.end()) {
1517 unsigned Reg = VMI->second;
1518 // If this is a PHI node, it may be split up into several MI PHI nodes
1519 // (in FunctionLoweringInfo::set).
1520 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1521 V->getType(), std::nullopt);
1522 if (RFV.occupiesMultipleRegs()) {
1523 // FIXME: We could potentially support variadic dbg_values here.
1524 if (IsVariadic)
1525 return false;
1526 unsigned Offset = 0;
1527 unsigned BitsToDescribe = 0;
1528 if (auto VarSize = Var->getSizeInBits())
1529 BitsToDescribe = *VarSize;
1530 if (auto Fragment = Expr->getFragmentInfo())
1531 BitsToDescribe = Fragment->SizeInBits;
1532 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1533 // Bail out if all bits are described already.
1534 if (Offset >= BitsToDescribe)
1535 break;
1536 // TODO: handle scalable vectors.
1537 unsigned RegisterSize = RegAndSize.second;
1538 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1539 ? BitsToDescribe - Offset
1540 : RegisterSize;
1541 auto FragmentExpr = DIExpression::createFragmentExpression(
1542 Expr, Offset, FragmentSize);
1543 if (!FragmentExpr)
1544 continue;
1546 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
1547 DAG.AddDbgValue(SDV, false);
1548 Offset += RegisterSize;
1549 }
1550 return true;
1551 }
1552 // We can use simple vreg locations for variadic dbg_values as well.
1553 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1554 continue;
1555 }
1556 // We failed to create a SDDbgOperand for V.
1557 return false;
1558 }
1559
1560 // We have created a SDDbgOperand for each Value in Values.
1561 // Should use Order instead of SDNodeOrder?
1562 assert(!LocationOps.empty());
1563 SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1564 /*IsIndirect=*/false, DbgLoc,
1565 SDNodeOrder, IsVariadic);
1566 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1567 return true;
1568}
1569
1571 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1572 for (auto &Pair : DanglingDebugInfoMap)
1573 for (auto &DDI : Pair.second)
1576}
1577
1578/// getCopyFromRegs - If there was virtual register allocated for the value V
1579/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1582 SDValue Result;
1583
1584 if (It != FuncInfo.ValueMap.end()) {
1585 Register InReg = It->second;
1586
1588 DAG.getDataLayout(), InReg, Ty,
1589 std::nullopt); // This is not an ABI copy.
1590 SDValue Chain = DAG.getEntryNode();
1591 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1592 V);
1593 resolveDanglingDebugInfo(V, Result);
1594 }
1595
1596 return Result;
1597}
1598
1599/// getValue - Return an SDValue for the given Value.
1601 // If we already have an SDValue for this value, use it. It's important
1602 // to do this first, so that we don't create a CopyFromReg if we already
1603 // have a regular SDValue.
1604 SDValue &N = NodeMap[V];
1605 if (N.getNode()) return N;
1606
1607 // If there's a virtual register allocated and initialized for this
1608 // value, use it.
1609 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1610 return copyFromReg;
1611
1612 // Otherwise create a new SDValue and remember it.
1613 SDValue Val = getValueImpl(V);
1614 NodeMap[V] = Val;
1616 return Val;
1617}
1618
1619/// getNonRegisterValue - Return an SDValue for the given Value, but
1620/// don't look in FuncInfo.ValueMap for a virtual register.
1622 // If we already have an SDValue for this value, use it.
1623 SDValue &N = NodeMap[V];
1624 if (N.getNode()) {
1625 if (isIntOrFPConstant(N)) {
1626 // Remove the debug location from the node as the node is about to be used
1627 // in a location which may differ from the original debug location. This
1628 // is relevant to Constant and ConstantFP nodes because they can appear
1629 // as constant expressions inside PHI nodes.
1630 N->setDebugLoc(DebugLoc());
1631 }
1632 return N;
1633 }
1634
1635 // Otherwise create a new SDValue and remember it.
1636 SDValue Val = getValueImpl(V);
1637 NodeMap[V] = Val;
1639 return Val;
1640}
1641
1642/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1643/// Create an SDValue for the given value.
1646
1647 if (const Constant *C = dyn_cast<Constant>(V)) {
1648 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1649
1650 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1651 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1652
1653 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1654 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1655
1656 if (isa<ConstantPointerNull>(C)) {
1657 unsigned AS = V->getType()->getPointerAddressSpace();
1658 return DAG.getConstant(0, getCurSDLoc(),
1659 TLI.getPointerTy(DAG.getDataLayout(), AS));
1660 }
1661
1662 if (match(C, m_VScale()))
1663 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1664
1665 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1666 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1667
1668 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1669 return DAG.getUNDEF(VT);
1670
1671 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1672 visit(CE->getOpcode(), *CE);
1673 SDValue N1 = NodeMap[V];
1674 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1675 return N1;
1676 }
1677
1678 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1680 for (const Use &U : C->operands()) {
1681 SDNode *Val = getValue(U).getNode();
1682 // If the operand is an empty aggregate, there are no values.
1683 if (!Val) continue;
1684 // Add each leaf value from the operand to the Constants list
1685 // to form a flattened list of all the values.
1686 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1687 Constants.push_back(SDValue(Val, i));
1688 }
1689
1691 }
1692
1693 if (const ConstantDataSequential *CDS =
1694 dyn_cast<ConstantDataSequential>(C)) {
1696 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1697 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1698 // Add each leaf value from the operand to the Constants list
1699 // to form a flattened list of all the values.
1700 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1701 Ops.push_back(SDValue(Val, i));
1702 }
1703
1704 if (isa<ArrayType>(CDS->getType()))
1705 return DAG.getMergeValues(Ops, getCurSDLoc());
1706 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1707 }
1708
1709 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1710 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1711 "Unknown struct or array constant!");
1712
1713 SmallVector<EVT, 4> ValueVTs;
1714 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1715 unsigned NumElts = ValueVTs.size();
1716 if (NumElts == 0)
1717 return SDValue(); // empty struct
1719 for (unsigned i = 0; i != NumElts; ++i) {
1720 EVT EltVT = ValueVTs[i];
1721 if (isa<UndefValue>(C))
1722 Constants[i] = DAG.getUNDEF(EltVT);
1723 else if (EltVT.isFloatingPoint())
1724 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1725 else
1726 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1727 }
1728
1730 }
1731
1732 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1733 return DAG.getBlockAddress(BA, VT);
1734
1735 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1736 return getValue(Equiv->getGlobalValue());
1737
1738 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1739 return getValue(NC->getGlobalValue());
1740
1741 VectorType *VecTy = cast<VectorType>(V->getType());
1742
1743 // Now that we know the number and type of the elements, get that number of
1744 // elements into the Ops array based on what kind of constant it is.
1745 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1747 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1748 for (unsigned i = 0; i != NumElements; ++i)
1749 Ops.push_back(getValue(CV->getOperand(i)));
1750
1751 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1752 }
1753
1754 if (isa<ConstantAggregateZero>(C)) {
1755 EVT EltVT =
1757
1758 SDValue Op;
1759 if (EltVT.isFloatingPoint())
1760 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1761 else
1762 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1763
1764 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1765 }
1766
1767 llvm_unreachable("Unknown vector constant");
1768 }
1769
1770 // If this is a static alloca, generate it as the frameindex instead of
1771 // computation.
1772 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1774 FuncInfo.StaticAllocaMap.find(AI);
1775 if (SI != FuncInfo.StaticAllocaMap.end())
1776 return DAG.getFrameIndex(
1777 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1778 }
1779
1780 // If this is an instruction which fast-isel has deferred, select it now.
1781 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1783
1784 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1785 Inst->getType(), std::nullopt);
1786 SDValue Chain = DAG.getEntryNode();
1787 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1788 }
1789
1790 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1791 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1792
1793 if (const auto *BB = dyn_cast<BasicBlock>(V))
1794 return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1795
1796 llvm_unreachable("Can't get register for value!");
1797}
1798
1799void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1801 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1802 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1803 bool IsSEH = isAsynchronousEHPersonality(Pers);
1804 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1805 if (!IsSEH)
1806 CatchPadMBB->setIsEHScopeEntry();
1807 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1808 if (IsMSVCCXX || IsCoreCLR)
1809 CatchPadMBB->setIsEHFuncletEntry();
1810}
1811
1812void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1813 // Update machine-CFG edge.
1814 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1815 FuncInfo.MBB->addSuccessor(TargetMBB);
1816 TargetMBB->setIsEHCatchretTarget(true);
1818
1820 bool IsSEH = isAsynchronousEHPersonality(Pers);
1821 if (IsSEH) {
1822 // If this is not a fall-through branch or optimizations are switched off,
1823 // emit the branch.
1824 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1826 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1827 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1828 return;
1829 }
1830
1831 // Figure out the funclet membership for the catchret's successor.
1832 // This will be used by the FuncletLayout pass to determine how to order the
1833 // BB's.
1834 // A 'catchret' returns to the outer scope's color.
1835 Value *ParentPad = I.getCatchSwitchParentPad();
1836 const BasicBlock *SuccessorColor;
1837 if (isa<ConstantTokenNone>(ParentPad))
1838 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1839 else
1840 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1841 assert(SuccessorColor && "No parent funclet for catchret!");
1842 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1843 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1844
1845 // Create the terminator node.
1847 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1848 DAG.getBasicBlock(SuccessorColorMBB));
1849 DAG.setRoot(Ret);
1850}
1851
1852void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1853 // Don't emit any special code for the cleanuppad instruction. It just marks
1854 // the start of an EH scope/funclet.
1857 if (Pers != EHPersonality::Wasm_CXX) {
1860 }
1861}
1862
1863// In wasm EH, even though a catchpad may not catch an exception if a tag does
1864// not match, it is OK to add only the first unwind destination catchpad to the
1865// successors, because there will be at least one invoke instruction within the
1866// catch scope that points to the next unwind destination, if one exists, so
1867// CFGSort cannot mess up with BB sorting order.
1868// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1869// call within them, and catchpads only consisting of 'catch (...)' have a
1870// '__cxa_end_catch' call within them, both of which generate invokes in case
1871// the next unwind destination exists, i.e., the next unwind destination is not
1872// the caller.)
1873//
1874// Having at most one EH pad successor is also simpler and helps later
1875// transformations.
1876//
1877// For example,
1878// current:
1879// invoke void @foo to ... unwind label %catch.dispatch
1880// catch.dispatch:
1881// %0 = catchswitch within ... [label %catch.start] unwind label %next
1882// catch.start:
1883// ...
1884// ... in this BB or some other child BB dominated by this BB there will be an
1885// invoke that points to 'next' BB as an unwind destination
1886//
1887// next: ; We don't need to add this to 'current' BB's successor
1888// ...
1890 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1891 BranchProbability Prob,
1892 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1893 &UnwindDests) {
1894 while (EHPadBB) {
1895 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1896 if (isa<CleanupPadInst>(Pad)) {
1897 // Stop on cleanup pads.
1898 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1899 UnwindDests.back().first->setIsEHScopeEntry();
1900 break;
1901 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1902 // Add the catchpad handlers to the possible destinations. We don't
1903 // continue to the unwind destination of the catchswitch for wasm.
1904 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1905 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1906 UnwindDests.back().first->setIsEHScopeEntry();
1907 }
1908 break;
1909 } else {
1910 continue;
1911 }
1912 }
1913}
1914
1915/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1916/// many places it could ultimately go. In the IR, we have a single unwind
1917/// destination, but in the machine CFG, we enumerate all the possible blocks.
1918/// This function skips over imaginary basic blocks that hold catchswitch
1919/// instructions, and finds all the "real" machine
1920/// basic block destinations. As those destinations may not be successors of
1921/// EHPadBB, here we also calculate the edge probability to those destinations.
1922/// The passed-in Prob is the edge probability to EHPadBB.
1924 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1925 BranchProbability Prob,
1926 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1927 &UnwindDests) {
1928 EHPersonality Personality =
1930 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1931 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1932 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1933 bool IsSEH = isAsynchronousEHPersonality(Personality);
1934
1935 if (IsWasmCXX) {
1936 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1937 assert(UnwindDests.size() <= 1 &&
1938 "There should be at most one unwind destination for wasm");
1939 return;
1940 }
1941
1942 while (EHPadBB) {
1943 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1944 BasicBlock *NewEHPadBB = nullptr;
1945 if (isa<LandingPadInst>(Pad)) {
1946 // Stop on landingpads. They are not funclets.
1947 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1948 break;
1949 } else if (isa<CleanupPadInst>(Pad)) {
1950 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1951 // personalities.
1952 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1953 UnwindDests.back().first->setIsEHScopeEntry();
1954 UnwindDests.back().first->setIsEHFuncletEntry();
1955 break;
1956 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1957 // Add the catchpad handlers to the possible destinations.
1958 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1959 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1960 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1961 if (IsMSVCCXX || IsCoreCLR)
1962 UnwindDests.back().first->setIsEHFuncletEntry();
1963 if (!IsSEH)
1964 UnwindDests.back().first->setIsEHScopeEntry();
1965 }
1966 NewEHPadBB = CatchSwitch->getUnwindDest();
1967 } else {
1968 continue;
1969 }
1970
1971 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1972 if (BPI && NewEHPadBB)
1973 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1974 EHPadBB = NewEHPadBB;
1975 }
1976}
1977
1978void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1979 // Update successor info.
1981 auto UnwindDest = I.getUnwindDest();
1983 BranchProbability UnwindDestProb =
1984 (BPI && UnwindDest)
1985 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1987 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1988 for (auto &UnwindDest : UnwindDests) {
1989 UnwindDest.first->setIsEHPad();
1990 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1991 }
1993
1994 // Create the terminator node.
1995 SDValue Ret =
1997 DAG.setRoot(Ret);
1998}
1999
2000void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2001 report_fatal_error("visitCatchSwitch not yet implemented!");
2002}
2003
2004void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2006 auto &DL = DAG.getDataLayout();
2007 SDValue Chain = getControlRoot();
2010
2011 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2012 // lower
2013 //
2014 // %val = call <ty> @llvm.experimental.deoptimize()
2015 // ret <ty> %val
2016 //
2017 // differently.
2018 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2020 return;
2021 }
2022
2023 if (!FuncInfo.CanLowerReturn) {
2024 unsigned DemoteReg = FuncInfo.DemoteRegister;
2025 const Function *F = I.getParent()->getParent();
2026
2027 // Emit a store of the return value through the virtual register.
2028 // Leave Outs empty so that LowerReturn won't try to load return
2029 // registers the usual way.
2030 SmallVector<EVT, 1> PtrValueVTs;
2031 ComputeValueVTs(TLI, DL,
2032 PointerType::get(F->getContext(),
2034 PtrValueVTs);
2035
2036 SDValue RetPtr =
2037 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2038 SDValue RetOp = getValue(I.getOperand(0));
2039
2040 SmallVector<EVT, 4> ValueVTs, MemVTs;
2042 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2043 &Offsets, 0);
2044 unsigned NumValues = ValueVTs.size();
2045
2046 SmallVector<SDValue, 4> Chains(NumValues);
2047 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2048 for (unsigned i = 0; i != NumValues; ++i) {
2049 // An aggregate return value cannot wrap around the address space, so
2050 // offsets to its parts don't wrap either.
2052 TypeSize::Fixed(Offsets[i]));
2053
2054 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2055 if (MemVTs[i] != ValueVTs[i])
2056 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2057 Chains[i] = DAG.getStore(
2058 Chain, getCurSDLoc(), Val,
2059 // FIXME: better loc info would be nice.
2061 commonAlignment(BaseAlign, Offsets[i]));
2062 }
2063
2065 MVT::Other, Chains);
2066 } else if (I.getNumOperands() != 0) {
2067 SmallVector<EVT, 4> ValueVTs;
2068 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2069 unsigned NumValues = ValueVTs.size();
2070 if (NumValues) {
2071 SDValue RetOp = getValue(I.getOperand(0));
2072
2073 const Function *F = I.getParent()->getParent();
2074
2075 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2076 I.getOperand(0)->getType(), F->getCallingConv(),
2077 /*IsVarArg*/ false, DL);
2078
2079 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2080 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2081 ExtendKind = ISD::SIGN_EXTEND;
2082 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2083 ExtendKind = ISD::ZERO_EXTEND;
2084
2085 LLVMContext &Context = F->getContext();
2086 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2087
2088 for (unsigned j = 0; j != NumValues; ++j) {
2089 EVT VT = ValueVTs[j];
2090
2091 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2092 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2093
2094 CallingConv::ID CC = F->getCallingConv();
2095
2096 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2097 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2098 SmallVector<SDValue, 4> Parts(NumParts);
2100 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2101 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2102
2103 // 'inreg' on function refers to return value
2105 if (RetInReg)
2106 Flags.setInReg();
2107
2108 if (I.getOperand(0)->getType()->isPointerTy()) {
2109 Flags.setPointer();
2110 Flags.setPointerAddrSpace(
2111 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2112 }
2113
2114 if (NeedsRegBlock) {
2115 Flags.setInConsecutiveRegs();
2116 if (j == NumValues - 1)
2117 Flags.setInConsecutiveRegsLast();
2118 }
2119
2120 // Propagate extension type if any
2121 if (ExtendKind == ISD::SIGN_EXTEND)
2122 Flags.setSExt();
2123 else if (ExtendKind == ISD::ZERO_EXTEND)
2124 Flags.setZExt();
2125
2126 for (unsigned i = 0; i < NumParts; ++i) {
2127 Outs.push_back(ISD::OutputArg(Flags,
2128 Parts[i].getValueType().getSimpleVT(),
2129 VT, /*isfixed=*/true, 0, 0));
2130 OutVals.push_back(Parts[i]);
2131 }
2132 }
2133 }
2134 }
2135
2136 // Push in swifterror virtual register as the last element of Outs. This makes
2137 // sure swifterror virtual register will be returned in the swifterror
2138 // physical register.
2139 const Function *F = I.getParent()->getParent();
2140 if (TLI.supportSwiftError() &&
2141 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2142 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2144 Flags.setSwiftError();
2146 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2147 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2148 // Create SDNode for the swifterror virtual register.
2149 OutVals.push_back(
2152 EVT(TLI.getPointerTy(DL))));
2153 }
2154
2155 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2156 CallingConv::ID CallConv =
2159 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2160
2161 // Verify that the target's LowerReturn behaved as expected.
2162 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2163 "LowerReturn didn't return a valid chain!");
2164
2165 // Update the DAG with the new chain value resulting from return lowering.
2166 DAG.setRoot(Chain);
2167}
2168
2169/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2170/// created for it, emit nodes to copy the value into the virtual
2171/// registers.
2173 // Skip empty types
2174 if (V->getType()->isEmptyTy())
2175 return;
2176
2178 if (VMI != FuncInfo.ValueMap.end()) {
2179 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2180 "Unused value assigned virtual registers!");
2181 CopyValueToVirtualRegister(V, VMI->second);
2182 }
2183}
2184
2185/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2186/// the current basic block, add it to ValueMap now so that we'll get a
2187/// CopyTo/FromReg.
2189 // No need to export constants.
2190 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2191
2192 // Already exported?
2193 if (FuncInfo.isExportedInst(V)) return;
2194
2197}
2198
2200 const BasicBlock *FromBB) {
2201 // The operands of the setcc have to be in this block. We don't know
2202 // how to export them from some other block.
2203 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2204 // Can export from current BB.
2205 if (VI->getParent() == FromBB)
2206 return true;
2207
2208 // Is already exported, noop.
2209 return FuncInfo.isExportedInst(V);
2210 }
2211
2212 // If this is an argument, we can export it if the BB is the entry block or
2213 // if it is already exported.
2214 if (isa<Argument>(V)) {
2215 if (FromBB->isEntryBlock())
2216 return true;
2217
2218 // Otherwise, can only export this if it is already exported.
2219 return FuncInfo.isExportedInst(V);
2220 }
2221
2222 // Otherwise, constants can always be exported.
2223 return true;
2224}
2225
2226/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2228SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2229 const MachineBasicBlock *Dst) const {
2231 const BasicBlock *SrcBB = Src->getBasicBlock();
2232 const BasicBlock *DstBB = Dst->getBasicBlock();
2233 if (!BPI) {
2234 // If BPI is not available, set the default probability as 1 / N, where N is
2235 // the number of successors.
2236 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2237 return BranchProbability(1, SuccSize);
2238 }
2239 return BPI->getEdgeProbability(SrcBB, DstBB);
2240}
2241
2242void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2243 MachineBasicBlock *Dst,
2244 BranchProbability Prob) {
2245 if (!FuncInfo.BPI)
2246 Src->addSuccessorWithoutProb(Dst);
2247 else {
2248 if (Prob.isUnknown())
2249 Prob = getEdgeProbability(Src, Dst);
2250 Src->addSuccessor(Dst, Prob);
2251 }
2252}
2253
2254static bool InBlock(const Value *V, const BasicBlock *BB) {
2255 if (const Instruction *I = dyn_cast<Instruction>(V))
2256 return I->getParent() == BB;
2257 return true;
2258}
2259
2260/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2261/// This function emits a branch and is used at the leaves of an OR or an
2262/// AND operator tree.
2263void
2266 MachineBasicBlock *FBB,
2267 MachineBasicBlock *CurBB,
2268 MachineBasicBlock *SwitchBB,
2269 BranchProbability TProb,
2270 BranchProbability FProb,
2271 bool InvertCond) {
2272 const BasicBlock *BB = CurBB->getBasicBlock();
2273
2274 // If the leaf of the tree is a comparison, merge the condition into
2275 // the caseblock.
2276 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2277 // The operands of the cmp have to be in this block. We don't know
2278 // how to export them from some other block. If this is the first block
2279 // of the sequence, no exporting is needed.
2280 if (CurBB == SwitchBB ||
2281 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2282 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2283 ISD::CondCode Condition;
2284 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2285 ICmpInst::Predicate Pred =
2286 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2287 Condition = getICmpCondCode(Pred);
2288 } else {
2289 const FCmpInst *FC = cast<FCmpInst>(Cond);
2290 FCmpInst::Predicate Pred =
2291 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2292 Condition = getFCmpCondCode(Pred);
2293 if (TM.Options.NoNaNsFPMath)
2294 Condition = getFCmpCodeWithoutNaN(Condition);
2295 }
2296
2297 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2298 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2299 SL->SwitchCases.push_back(CB);
2300 return;
2301 }
2302 }
2303
2304 // Create a CaseBlock record representing this branch.
2305 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2307 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2308 SL->SwitchCases.push_back(CB);
2309}
2310
2313 MachineBasicBlock *FBB,
2314 MachineBasicBlock *CurBB,
2315 MachineBasicBlock *SwitchBB,
2317 BranchProbability TProb,
2318 BranchProbability FProb,
2319 bool InvertCond) {
2320 // Skip over not part of the tree and remember to invert op and operands at
2321 // next level.
2322 Value *NotCond;
2323 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2324 InBlock(NotCond, CurBB->getBasicBlock())) {
2325 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2326 !InvertCond);
2327 return;
2328 }
2329
2330 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2331 const Value *BOpOp0, *BOpOp1;
2332 // Compute the effective opcode for Cond, taking into account whether it needs
2333 // to be inverted, e.g.
2334 // and (not (or A, B)), C
2335 // gets lowered as
2336 // and (and (not A, not B), C)
2338 if (BOp) {
2339 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2340 ? Instruction::And
2341 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2342 ? Instruction::Or
2344 if (InvertCond) {
2345 if (BOpc == Instruction::And)
2346 BOpc = Instruction::Or;
2347 else if (BOpc == Instruction::Or)
2348 BOpc = Instruction::And;
2349 }
2350 }
2351
2352 // If this node is not part of the or/and tree, emit it as a branch.
2353 // Note that all nodes in the tree should have same opcode.
2354 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2355 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2356 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2357 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2358 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2359 TProb, FProb, InvertCond);
2360 return;
2361 }
2362
2363 // Create TmpBB after CurBB.
2364 MachineFunction::iterator BBI(CurBB);
2367 CurBB->getParent()->insert(++BBI, TmpBB);
2368
2369 if (Opc == Instruction::Or) {
2370 // Codegen X | Y as:
2371 // BB1:
2372 // jmp_if_X TBB
2373 // jmp TmpBB
2374 // TmpBB:
2375 // jmp_if_Y TBB
2376 // jmp FBB
2377 //
2378
2379 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2380 // The requirement is that
2381 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2382 // = TrueProb for original BB.
2383 // Assuming the original probabilities are A and B, one choice is to set
2384 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2385 // A/(1+B) and 2B/(1+B). This choice assumes that
2386 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2387 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2388 // TmpBB, but the math is more complicated.
2389
2390 auto NewTrueProb = TProb / 2;
2391 auto NewFalseProb = TProb / 2 + FProb;
2392 // Emit the LHS condition.
2393 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2394 NewFalseProb, InvertCond);
2395
2396 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2397 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2398 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2399 // Emit the RHS condition into TmpBB.
2400 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2401 Probs[1], InvertCond);
2402 } else {
2403 assert(Opc == Instruction::And && "Unknown merge op!");
2404 // Codegen X & Y as:
2405 // BB1:
2406 // jmp_if_X TmpBB
2407 // jmp FBB
2408 // TmpBB:
2409 // jmp_if_Y TBB
2410 // jmp FBB
2411 //
2412 // This requires creation of TmpBB after CurBB.
2413
2414 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2415 // The requirement is that
2416 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2417 // = FalseProb for original BB.
2418 // Assuming the original probabilities are A and B, one choice is to set
2419 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2420 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2421 // TrueProb for BB1 * FalseProb for TmpBB.
2422
2423 auto NewTrueProb = TProb + FProb / 2;
2424 auto NewFalseProb = FProb / 2;
2425 // Emit the LHS condition.
2426 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2427 NewFalseProb, InvertCond);
2428
2429 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2430 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2431 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2432 // Emit the RHS condition into TmpBB.
2433 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2434 Probs[1], InvertCond);
2435 }
2436}
2437
2438/// If the set of cases should be emitted as a series of branches, return true.
2439/// If we should emit this as a bunch of and/or'd together conditions, return
2440/// false.
2441bool
2442SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2443 if (Cases.size() != 2) return true;
2444
2445 // If this is two comparisons of the same values or'd or and'd together, they
2446 // will get folded into a single comparison, so don't emit two blocks.
2447 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2448 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2449 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2450 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2451 return false;
2452 }
2453
2454 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2455 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2456 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2457 Cases[0].CC == Cases[1].CC &&
2458 isa<Constant>(Cases[0].CmpRHS) &&
2459 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2460 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2461 return false;
2462 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2463 return false;
2464 }
2465
2466 return true;
2467}
2468
2469void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2471
2472 // Update machine-CFG edges.
2473 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2474
2475 if (I.isUnconditional()) {
2476 // Update machine-CFG edges.
2477 BrMBB->addSuccessor(Succ0MBB);
2478
2479 // If this is not a fall-through branch or optimizations are switched off,
2480 // emit the branch.
2481 if (Succ0MBB != NextBlock(BrMBB) ||
2483 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2484 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2485 setValue(&I, Br);
2486 DAG.setRoot(Br);
2487 }
2488
2489 return;
2490 }
2491
2492 // If this condition is one of the special cases we handle, do special stuff
2493 // now.
2494 const Value *CondVal = I.getCondition();
2495 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2496
2497 // If this is a series of conditions that are or'd or and'd together, emit
2498 // this as a sequence of branches instead of setcc's with and/or operations.
2499 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2500 // unpredictable branches, and vector extracts because those jumps are likely
2501 // expensive for any target), this should improve performance.
2502 // For example, instead of something like:
2503 // cmp A, B
2504 // C = seteq
2505 // cmp D, E
2506 // F = setle
2507 // or C, F
2508 // jnz foo
2509 // Emit:
2510 // cmp A, B
2511 // je foo
2512 // cmp D, E
2513 // jle foo
2514 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2515 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2516 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2517 Value *Vec;
2518 const Value *BOp0, *BOp1;
2520 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2521 Opcode = Instruction::And;
2522 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2523 Opcode = Instruction::Or;
2524
2525 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2526 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2527 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2528 getEdgeProbability(BrMBB, Succ0MBB),
2529 getEdgeProbability(BrMBB, Succ1MBB),
2530 /*InvertCond=*/false);
2531 // If the compares in later blocks need to use values not currently
2532 // exported from this block, export them now. This block should always
2533 // be the first entry.
2534 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2535
2536 // Allow some cases to be rejected.
2537 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2538 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2539 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2540 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2541 }
2542
2543 // Emit the branch for this block.
2544 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2545 SL->SwitchCases.erase(SL->SwitchCases.begin());
2546 return;
2547 }
2548
2549 // Okay, we decided not to do this, remove any inserted MBB's and clear
2550 // SwitchCases.
2551 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2552 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2553
2554 SL->SwitchCases.clear();
2555 }
2556 }
2557
2558 // Create a CaseBlock record representing this branch.
2560 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2561
2562 // Use visitSwitchCase to actually insert the fast branch sequence for this
2563 // cond branch.
2564 visitSwitchCase(CB, BrMBB);
2565}
2566
2567/// visitSwitchCase - Emits the necessary code to represent a single node in
2568/// the binary search tree resulting from lowering a switch instruction.
2570 MachineBasicBlock *SwitchBB) {
2571 SDValue Cond;
2572 SDValue CondLHS = getValue(CB.CmpLHS);
2573 SDLoc dl = CB.DL;
2574
2575 if (CB.CC == ISD::SETTRUE) {
2576 // Branch or fall through to TrueBB.
2577 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2578 SwitchBB->normalizeSuccProbs();
2579 if (CB.TrueBB != NextBlock(SwitchBB)) {
2580 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2581 DAG.getBasicBlock(CB.TrueBB)));
2582 }
2583 return;
2584 }
2585
2586 auto &TLI = DAG.getTargetLoweringInfo();
2587 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2588
2589 // Build the setcc now.
2590 if (!CB.CmpMHS) {
2591 // Fold "(X == true)" to X and "(X == false)" to !X to
2592 // handle common cases produced by branch lowering.
2593 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2594 CB.CC == ISD::SETEQ)
2595 Cond = CondLHS;
2596 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2597 CB.CC == ISD::SETEQ) {
2598 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2599 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2600 } else {
2601 SDValue CondRHS = getValue(CB.CmpRHS);
2602
2603 // If a pointer's DAG type is larger than its memory type then the DAG
2604 // values are zero-extended. This breaks signed comparisons so truncate
2605 // back to the underlying type before doing the compare.
2606 if (CondLHS.getValueType() != MemVT) {
2607 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2608 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2609 }
2610 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2611 }
2612 } else {
2613 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2614
2615 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2616 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2617
2618 SDValue CmpOp = getValue(CB.CmpMHS);
2619 EVT VT = CmpOp.getValueType();
2620
2621 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2622 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2623 ISD::SETLE);
2624 } else {
2625 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2626 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2627 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2628 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2629 }
2630 }
2631
2632 // Update successor info
2633 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2634 // TrueBB and FalseBB are always different unless the incoming IR is
2635 // degenerate. This only happens when running llc on weird IR.
2636 if (CB.TrueBB != CB.FalseBB)
2637 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2638 SwitchBB->normalizeSuccProbs();
2639
2640 // If the lhs block is the next block, invert the condition so that we can
2641 // fall through to the lhs instead of the rhs block.
2642 if (CB.TrueBB == NextBlock(SwitchBB)) {
2643 std::swap(CB.TrueBB, CB.FalseBB);
2644 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2645 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2646 }
2647
2648 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2649 MVT::Other, getControlRoot(), Cond,
2651
2652 setValue(CurInst, BrCond);
2653
2654 // Insert the false branch. Do this even if it's a fall through branch,
2655 // this makes it easier to do DAG optimizations which require inverting
2656 // the branch condition.
2657 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2659
2660 DAG.setRoot(BrCond);
2661}
2662
2663/// visitJumpTable - Emit JumpTable node in the current MBB
2665 // Emit the code for the jump table
2666 assert(JT.Reg != -1U && "Should lower JT Header first!");
2669 JT.Reg, PTy);
2670 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2671 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2672 MVT::Other, Index.getValue(1),
2673 Table, Index);
2674 DAG.setRoot(BrJumpTable);
2675}
2676
2677/// visitJumpTableHeader - This function emits necessary code to produce index
2678/// in the JumpTable from switch case.
2680 JumpTableHeader &JTH,
2681 MachineBasicBlock *SwitchBB) {
2682 SDLoc dl = getCurSDLoc();
2683
2684 // Subtract the lowest switch case value from the value being switched on.
2685 SDValue SwitchOp = getValue(JTH.SValue);
2686 EVT VT = SwitchOp.getValueType();
2687 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2688 DAG.getConstant(JTH.First, dl, VT));
2689
2690 // The SDNode we just created, which holds the value being switched on minus
2691 // the smallest case value, needs to be copied to a virtual register so it
2692 // can be used as an index into the jump table in a subsequent basic block.
2693 // This value may be smaller or larger than the target's pointer type, and
2694 // therefore require extension or truncating.
2696 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2697
2698 unsigned JumpTableReg =
2700 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2701 JumpTableReg, SwitchOp);
2702 JT.Reg = JumpTableReg;
2703
2704 if (!JTH.FallthroughUnreachable) {
2705 // Emit the range check for the jump table, and branch to the default block
2706 // for the switch statement if the value being switched on exceeds the
2707 // largest case in the switch.
2708 SDValue CMP = DAG.getSetCC(
2710 Sub.getValueType()),
2711 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2712
2713 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2714 MVT::Other, CopyTo, CMP,
2715 DAG.getBasicBlock(JT.Default));
2716
2717 // Avoid emitting unnecessary branches to the next block.
2718 if (JT.MBB != NextBlock(SwitchBB))
2719 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2720 DAG.getBasicBlock(JT.MBB));
2721
2722 DAG.setRoot(BrCond);
2723 } else {
2724 // Avoid emitting unnecessary branches to the next block.
2725 if (JT.MBB != NextBlock(SwitchBB))
2726 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2727 DAG.getBasicBlock(JT.MBB)));
2728 else
2729 DAG.setRoot(CopyTo);
2730 }
2731}
2732
2733/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2734/// variable if there exists one.
2736 SDValue &Chain) {
2737 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2738 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2739 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2743 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2744 if (Global) {
2745 MachinePointerInfo MPInfo(Global);
2749 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2750 DAG.setNodeMemRefs(Node, {MemRef});
2751 }
2752 if (PtrTy != PtrMemTy)
2753 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2754 return SDValue(Node, 0);
2755}
2756
2757/// Codegen a new tail for a stack protector check ParentMBB which has had its
2758/// tail spliced into a stack protector check success bb.
2759///
2760/// For a high level explanation of how this fits into the stack protector
2761/// generation see the comment on the declaration of class
2762/// StackProtectorDescriptor.
2764 MachineBasicBlock *ParentBB) {
2765
2766 // First create the loads to the guard/stack slot for the comparison.
2768 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2769 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2770
2771 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2772 int FI = MFI.getStackProtectorIndex();
2773
2774 SDValue Guard;
2775 SDLoc dl = getCurSDLoc();
2776 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2777 const Module &M = *ParentBB->getParent()->getFunction().getParent();
2778 Align Align =
2779 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
2780
2781 // Generate code to load the content of the guard slot.
2782 SDValue GuardVal = DAG.getLoad(
2783 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2786
2787 if (TLI.useStackGuardXorFP())
2788 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2789
2790 // Retrieve guard check function, nullptr if instrumentation is inlined.
2791 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2792 // The target provides a guard check function to validate the guard value.
2793 // Generate a call to that function with the content of the guard slot as
2794 // argument.
2795 FunctionType *FnTy = GuardCheckFn->getFunctionType();
2796 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2797
2800 Entry.Node = GuardVal;
2801 Entry.Ty = FnTy->getParamType(0);
2802 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2803 Entry.IsInReg = true;
2804 Args.push_back(Entry);
2805
2809 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2810 getValue(GuardCheckFn), std::move(Args));
2811
2812 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2813 DAG.setRoot(Result.second);
2814 return;
2815 }
2816
2817 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2818 // Otherwise, emit a volatile load to retrieve the stack guard value.
2819 SDValue Chain = DAG.getEntryNode();
2820 if (TLI.useLoadStackGuardNode()) {
2821 Guard = getLoadStackGuard(DAG, dl, Chain);
2822 } else {
2823 const Value *IRGuard = TLI.getSDagStackGuard(M);
2824 SDValue GuardPtr = getValue(IRGuard);
2825
2826 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2827 MachinePointerInfo(IRGuard, 0), Align,
2829 }
2830
2831 // Perform the comparison via a getsetcc.
2833 *DAG.getContext(),
2834 Guard.getValueType()),
2835 Guard, GuardVal, ISD::SETNE);
2836
2837 // If the guard/stackslot do not equal, branch to failure MBB.
2838 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2839 MVT::Other, GuardVal.getOperand(0),
2840 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2841 // Otherwise branch to success MBB.
2842 SDValue Br = DAG.getNode(ISD::BR, dl,
2843 MVT::Other, BrCond,
2845
2846 DAG.setRoot(Br);
2847}
2848
2849/// Codegen the failure basic block for a stack protector check.
2850///
2851/// A failure stack protector machine basic block consists simply of a call to
2852/// __stack_chk_fail().
2853///
2854/// For a high level explanation of how this fits into the stack protector
2855/// generation see the comment on the declaration of class
2856/// StackProtectorDescriptor.
2857void
2861 CallOptions.setDiscardResult(true);
2862 SDValue Chain =
2863 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2864 std::nullopt, CallOptions, getCurSDLoc())
2865 .second;
2866 // On PS4/PS5, the "return address" must still be within the calling
2867 // function, even if it's at the very end, so emit an explicit TRAP here.
2868 // Passing 'true' for doesNotReturn above won't generate the trap for us.
2869 if (TM.getTargetTriple().isPS())
2870 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2871 // WebAssembly needs an unreachable instruction after a non-returning call,
2872 // because the function return type can be different from __stack_chk_fail's
2873 // return type (void).
2874 if (TM.getTargetTriple().isWasm())
2875 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2876
2877 DAG.setRoot(Chain);
2878}
2879
2880/// visitBitTestHeader - This function emits necessary code to produce value
2881/// suitable for "bit tests"
2883 MachineBasicBlock *SwitchBB) {
2884 SDLoc dl = getCurSDLoc();
2885
2886 // Subtract the minimum value.
2887 SDValue SwitchOp = getValue(B.SValue);
2888 EVT VT = SwitchOp.getValueType();
2889 SDValue RangeSub =
2890 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2891
2892 // Determine the type of the test operands.
2894 bool UsePtrType = false;
2895 if (!TLI.isTypeLegal(VT)) {
2896 UsePtrType = true;
2897 } else {
2898 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2899 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2900 // Switch table case range are encoded into series of masks.
2901 // Just use pointer type, it's guaranteed to fit.
2902 UsePtrType = true;
2903 break;
2904 }
2905 }
2906 SDValue Sub = RangeSub;
2907 if (UsePtrType) {
2908 VT = TLI.getPointerTy(DAG.getDataLayout());
2909 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2910 }
2911
2912 B.RegVT = VT.getSimpleVT();
2913 B.Reg = FuncInfo.CreateReg(B.RegVT);
2914 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2915
2916 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2917
2918 if (!B.FallthroughUnreachable)
2919 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2920 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2921 SwitchBB->normalizeSuccProbs();
2922
2923 SDValue Root = CopyTo;
2924 if (!B.FallthroughUnreachable) {
2925 // Conditional branch to the default block.
2926 SDValue RangeCmp = DAG.getSetCC(dl,
2928 RangeSub.getValueType()),
2929 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2930 ISD::SETUGT);
2931
2932 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2933 DAG.getBasicBlock(B.Default));
2934 }
2935
2936 // Avoid emitting unnecessary branches to the next block.
2937 if (MBB != NextBlock(SwitchBB))
2938 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2939
2940 DAG.setRoot(Root);
2941}
2942
2943/// visitBitTestCase - this function produces one "bit test"
2945 MachineBasicBlock* NextMBB,
2946 BranchProbability BranchProbToNext,
2947 unsigned Reg,
2948 BitTestCase &B,
2949 MachineBasicBlock *SwitchBB) {
2950 SDLoc dl = getCurSDLoc();
2951 MVT VT = BB.RegVT;
2952 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2953 SDValue Cmp;
2954 unsigned PopCount = llvm::popcount(B.Mask);
2956 if (PopCount == 1) {
2957 // Testing for a single bit; just compare the shift count with what it
2958 // would need to be to shift a 1 bit in that position.
2959 Cmp = DAG.getSetCC(
2961 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
2962 ISD::SETEQ);
2963 } else if (PopCount == BB.Range) {
2964 // There is only one zero bit in the range, test for it directly.
2965 Cmp = DAG.getSetCC(
2967 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
2968 } else {
2969 // Make desired shift
2970 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2971 DAG.getConstant(1, dl, VT), ShiftOp);
2972
2973 // Emit bit tests and jumps
2974 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2975 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2976 Cmp = DAG.getSetCC(
2978 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2979 }
2980
2981 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2982 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2983 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2984 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2985 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2986 // one as they are relative probabilities (and thus work more like weights),
2987 // and hence we need to normalize them to let the sum of them become one.
2988 SwitchBB->normalizeSuccProbs();
2989
2990 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2991 MVT::Other, getControlRoot(),
2992 Cmp, DAG.getBasicBlock(B.TargetBB));
2993
2994 // Avoid emitting unnecessary branches to the next block.
2995 if (NextMBB != NextBlock(SwitchBB))
2996 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2997 DAG.getBasicBlock(NextMBB));
2998
2999 DAG.setRoot(BrAnd);
3000}
3001
3002void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3003 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3004
3005 // Retrieve successors. Look through artificial IR level blocks like
3006 // catchswitch for successors.
3007 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
3008 const BasicBlock *EHPadBB = I.getSuccessor(1);
3009 MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
3010
3011 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3012 // have to do anything here to lower funclet bundles.
3013 assert(!I.hasOperandBundlesOtherThan(
3014 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3015 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3016 LLVMContext::OB_cfguardtarget,
3017 LLVMContext::OB_clang_arc_attachedcall}) &&
3018 "Cannot lower invokes with arbitrary operand bundles yet!");
3019
3020 const Value *Callee(I.getCalledOperand());
3021 const Function *Fn = dyn_cast<Function>(Callee);
3022 if (isa<InlineAsm>(Callee))
3023 visitInlineAsm(I, EHPadBB);
3024 else if (Fn && Fn->isIntrinsic()) {
3025 switch (Fn->getIntrinsicID()) {
3026 default:
3027 llvm_unreachable("Cannot invoke this intrinsic");
3028 case Intrinsic::donothing:
3029 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3030 case Intrinsic::seh_try_begin:
3031 case Intrinsic::seh_scope_begin:
3032 case Intrinsic::seh_try_end:
3033 case Intrinsic::seh_scope_end:
3034 if (EHPadMBB)
3035 // a block referenced by EH table
3036 // so dtor-funclet not removed by opts
3037 EHPadMBB->setMachineBlockAddressTaken();
3038 break;
3039 case Intrinsic::experimental_patchpoint_void:
3040 case Intrinsic::experimental_patchpoint_i64:
3041 visitPatchpoint(I, EHPadBB);
3042 break;
3043 case Intrinsic::experimental_gc_statepoint:
3044 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3045 break;
3046 case Intrinsic::wasm_rethrow: {
3047 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3048 // special because it can be invoked, so we manually lower it to a DAG
3049 // node here.
3051 Ops.push_back(getRoot()); // inchain
3053 Ops.push_back(
3054 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3056 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3058 break;
3059 }
3060 }
3061 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
3062 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3063 // Eventually we will support lowering the @llvm.experimental.deoptimize
3064 // intrinsic, and right now there are no plans to support other intrinsics
3065 // with deopt state.
3066 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3067 } else {
3068 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3069 }
3070
3071 // If the value of the invoke is used outside of its defining block, make it
3072 // available as a virtual register.
3073 // We already took care of the exported value for the statepoint instruction
3074 // during call to the LowerStatepoint.
3075 if (!isa<GCStatepointInst>(I)) {
3077 }
3078
3081 BranchProbability EHPadBBProb =
3082 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3084 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3085
3086 // Update successor info.
3087 addSuccessorWithProb(InvokeMBB, Return);
3088 for (auto &UnwindDest : UnwindDests) {
3089 UnwindDest.first->setIsEHPad();
3090 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3091 }
3092 InvokeMBB->normalizeSuccProbs();
3093
3094 // Drop into normal successor.
3096 DAG.getBasicBlock(Return)));
3097}
3098
3099void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3100 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3101
3102 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3103 // have to do anything here to lower funclet bundles.
3104 assert(!I.hasOperandBundlesOtherThan(
3105 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3106 "Cannot lower callbrs with arbitrary operand bundles yet!");
3107
3108 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3109 visitInlineAsm(I);
3111
3112 // Retrieve successors.
3114 Dests.insert(I.getDefaultDest());
3115 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
3116
3117 // Update successor info.
3118 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3119 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3120 BasicBlock *Dest = I.getIndirectDest(i);
3122 Target->setIsInlineAsmBrIndirectTarget();
3123 Target->setMachineBlockAddressTaken();
3124 Target->setLabelMustBeEmitted();
3125 // Don't add duplicate machine successors.
3126 if (Dests.insert(Dest).second)
3127 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3128 }
3129 CallBrMBB->normalizeSuccProbs();
3130
3131 // Drop into default successor.
3133 MVT::Other, getControlRoot(),
3134 DAG.getBasicBlock(Return)));
3135}
3136
3137void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3138 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3139}
3140
3141void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3143 "Call to landingpad not in landing pad!");
3144
3145 // If there aren't registers to copy the values into (e.g., during SjLj
3146 // exceptions), then don't bother to create these DAG nodes.
3148 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3149 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3150 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3151 return;
3152
3153 // If landingpad's return type is token type, we don't create DAG nodes
3154 // for its exception pointer and selector value. The extraction of exception
3155 // pointer or selector value from token type landingpads is not currently
3156 // supported.
3157 if (LP.getType()->isTokenTy())
3158 return;
3159
3160 SmallVector<EVT, 2> ValueVTs;
3161 SDLoc dl = getCurSDLoc();
3162 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3163 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3164
3165 // Get the two live-in registers as SDValues. The physregs have already been
3166 // copied into virtual registers.
3167 SDValue Ops[2];
3169 Ops[0] = DAG.getZExtOrTrunc(
3173 dl, ValueVTs[0]);
3174 } else {
3175 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3176 }
3177 Ops[1] = DAG.getZExtOrTrunc(
3181 dl, ValueVTs[1]);
3182
3183 // Merge into one.
3185 DAG.getVTList(ValueVTs), Ops);
3186 setValue(&LP, Res);
3187}
3188
3191 // Update JTCases.
3192 for (JumpTableBlock &JTB : SL->JTCases)
3193 if (JTB.first.HeaderBB == First)
3194 JTB.first.HeaderBB = Last;
3195
3196 // Update BitTestCases.
3197 for (BitTestBlock &BTB : SL->BitTestCases)
3198 if (BTB.Parent == First)
3199 BTB.Parent = Last;
3200}
3201
3202void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3203 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3204
3205 // Update machine-CFG edges with unique successors.
3207 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3208 BasicBlock *BB = I.getSuccessor(i);
3209 bool Inserted = Done.insert(BB).second;
3210 if (!Inserted)
3211 continue;
3212
3213 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3214 addSuccessorWithProb(IndirectBrMBB, Succ);
3215 }
3216 IndirectBrMBB->normalizeSuccProbs();
3217
3219 MVT::Other, getControlRoot(),
3220 getValue(I.getAddress())));
3221}
3222
3223void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3225 return;
3226
3227 // We may be able to ignore unreachable behind a noreturn call.
3229 const BasicBlock &BB = *I.getParent();
3230 if (&I != &BB.front()) {
3232 std::prev(BasicBlock::const_iterator(&I));
3233 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3234 if (Call->doesNotReturn())
3235 return;
3236 }
3237 }
3238 }
3239
3240 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3241}
3242
3243void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3245 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3246 Flags.copyFMF(*FPOp);
3247
3248 SDValue Op = getValue(I.getOperand(0));
3249 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3250 Op, Flags);
3251 setValue(&I, UnNodeValue);
3252}
3253
3254void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3256 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3257 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3258 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3259 }
3260 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3261 Flags.setExact(ExactOp->isExact());
3262 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3263 Flags.copyFMF(*FPOp);
3264
3265 SDValue Op1 = getValue(I.getOperand(0));
3266 SDValue Op2 = getValue(I.getOperand(1));
3267 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3268 Op1, Op2, Flags);
3269 setValue(&I, BinNodeValue);
3270}
3271
3272void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3273 SDValue Op1 = getValue(I.getOperand(0));
3274 SDValue Op2 = getValue(I.getOperand(1));
3275
3277 Op1.getValueType(), DAG.getDataLayout());
3278
3279 // Coerce the shift amount to the right type if we can. This exposes the
3280 // truncate or zext to optimization early.
3281 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3283 "Unexpected shift type");
3284 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3285 }
3286
3287 bool nuw = false;
3288 bool nsw = false;
3289 bool exact = false;
3290
3291 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3292
3293 if (const OverflowingBinaryOperator *OFBinOp =
3294 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3295 nuw = OFBinOp->hasNoUnsignedWrap();
3296 nsw = OFBinOp->hasNoSignedWrap();
3297 }
3298 if (const PossiblyExactOperator *ExactOp =
3299 dyn_cast<const PossiblyExactOperator>(&I))
3300 exact = ExactOp->isExact();
3301 }
3303 Flags.setExact(exact);
3304 Flags.setNoSignedWrap(nsw);
3305 Flags.setNoUnsignedWrap(nuw);
3306 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3307 Flags);
3308 setValue(&I, Res);
3309}
3310
3311void SelectionDAGBuilder::visitSDiv(const User &I) {
3312 SDValue Op1 = getValue(I.getOperand(0));
3313 SDValue Op2 = getValue(I.getOperand(1));
3314
3316 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3317 cast<PossiblyExactOperator>(&I)->isExact());
3319 Op2, Flags));
3320}
3321
3322void SelectionDAGBuilder::visitICmp(const User &I) {
3324 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3325 predicate = IC->getPredicate();
3326 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3327 predicate = ICmpInst::Predicate(IC->getPredicate());
3328 SDValue Op1 = getValue(I.getOperand(0));
3329 SDValue Op2 = getValue(I.getOperand(1));
3330 ISD::CondCode Opcode = getICmpCondCode(predicate);
3331
3332 auto &TLI = DAG.getTargetLoweringInfo();
3333 EVT MemVT =
3334 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3335
3336 // If a pointer's DAG type is larger than its memory type then the DAG values
3337 // are zero-extended. This breaks signed comparisons so truncate back to the
3338 // underlying type before doing the compare.
3339 if (Op1.getValueType() != MemVT) {
3340 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3341 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3342 }
3343
3345 I.getType());
3346 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3347}
3348
3349void SelectionDAGBuilder::visitFCmp(const User &I) {
3351 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3352 predicate = FC->getPredicate();
3353 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3354 predicate = FCmpInst::Predicate(FC->getPredicate());
3355 SDValue Op1 = getValue(I.getOperand(0));
3356 SDValue Op2 = getValue(I.getOperand(1));
3357
3358 ISD::CondCode Condition = getFCmpCondCode(predicate);
3359 auto *FPMO = cast<FPMathOperator>(&I);
3360 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3361 Condition = getFCmpCodeWithoutNaN(Condition);
3362
3364 Flags.copyFMF(*FPMO);
3365 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3366
3368 I.getType());
3369 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3370}
3371
3372// Check if the condition of the select has one use or two users that are both
3373// selects with the same condition.
3374static bool hasOnlySelectUsers(const Value *Cond) {
3375 return llvm::all_of(Cond->users(), [](const Value *V) {
3376 return isa<SelectInst>(V);
3377 });
3378}
3379
3380void SelectionDAGBuilder::visitSelect(const User &I) {
3381 SmallVector<EVT, 4> ValueVTs;
3383 ValueVTs);
3384 unsigned NumValues = ValueVTs.size();
3385 if (NumValues == 0) return;
3386
3387 SmallVector<SDValue, 4> Values(NumValues);
3388 SDValue Cond = getValue(I.getOperand(0));
3389 SDValue LHSVal = getValue(I.getOperand(1));
3390 SDValue RHSVal = getValue(I.getOperand(2));
3391 SmallVector<SDValue, 1> BaseOps(1, Cond);
3392 ISD::NodeType OpCode =
3393 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3394
3395 bool IsUnaryAbs = false;
3396 bool Negate = false;
3397
3399 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3400 Flags.copyFMF(*FPOp);
3401
3402 Flags.setUnpredictable(
3403 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3404
3405 // Min/max matching is only viable if all output VTs are the same.
3406 if (all_equal(ValueVTs)) {
3407 EVT VT = ValueVTs[0];
3408 LLVMContext &Ctx = *DAG.getContext();
3409 auto &TLI = DAG.getTargetLoweringInfo();
3410
3411 // We care about the legality of the operation after it has been type
3412 // legalized.
3413 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3414 VT = TLI.getTypeToTransformTo(Ctx, VT);
3415
3416 // If the vselect is legal, assume we want to leave this as a vector setcc +
3417 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3418 // min/max is legal on the scalar type.
3419 bool UseScalarMinMax = VT.isVector() &&
3421
3422 // ValueTracking's select pattern matching does not account for -0.0,
3423 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3424 // -0.0 is less than +0.0.
3425 Value *LHS, *RHS;
3426 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3428 switch (SPR.Flavor) {
3429 case SPF_UMAX: Opc = ISD::UMAX; break;
3430 case SPF_UMIN: Opc = ISD::UMIN; break;
3431 case SPF_SMAX: Opc = ISD::SMAX; break;
3432 case SPF_SMIN: Opc = ISD::SMIN; break;
3433 case SPF_FMINNUM:
3434 switch (SPR.NaNBehavior) {
3435 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3436 case SPNB_RETURNS_NAN: break;
3437 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3438 case SPNB_RETURNS_ANY:
3440 (UseScalarMinMax &&
3442 Opc = ISD::FMINNUM;
3443 break;
3444 }
3445 break;
3446 case SPF_FMAXNUM:
3447 switch (SPR.NaNBehavior) {
3448 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3449 case SPNB_RETURNS_NAN: break;
3450 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3451 case SPNB_RETURNS_ANY:
3453 (UseScalarMinMax &&
3455 Opc = ISD::FMAXNUM;
3456 break;
3457 }
3458 break;
3459 case SPF_NABS:
3460 Negate = true;
3461 [[fallthrough]];
3462 case SPF_ABS:
3463 IsUnaryAbs = true;
3464 Opc = ISD::ABS;
3465 break;
3466 default: break;
3467 }
3468
3469 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3470 (TLI.isOperationLegalOrCustom(Opc, VT) ||
3471 (UseScalarMinMax &&
3472 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3473 // If the underlying comparison instruction is used by any other
3474 // instruction, the consumed instructions won't be destroyed, so it is
3475 // not profitable to convert to a min/max.
3476 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3477 OpCode = Opc;
3478 LHSVal = getValue(LHS);
3479 RHSVal = getValue(RHS);
3480 BaseOps.clear();
3481 }
3482
3483 if (IsUnaryAbs) {
3484 OpCode = Opc;
3485 LHSVal = getValue(LHS);
3486 BaseOps.clear();
3487 }
3488 }
3489
3490 if (IsUnaryAbs) {
3491 for (unsigned i = 0; i != NumValues; ++i) {
3492 SDLoc dl = getCurSDLoc();
3493 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3494 Values[i] =
3495 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3496 if (Negate)
3497 Values[i] = DAG.getNegative(Values[i], dl, VT);
3498 }
3499 } else {
3500 for (unsigned i = 0; i != NumValues; ++i) {
3501 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3502 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3503 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3504 Values[i] = DAG.getNode(
3505 OpCode, getCurSDLoc(),
3506 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3507 }
3508 }
3509
3511 DAG.getVTList(ValueVTs), Values));
3512}
3513
3514void SelectionDAGBuilder::visitTrunc(const User &I) {
3515 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3516 SDValue N = getValue(I.getOperand(0));
3518 I.getType());
3520}
3521
3522void SelectionDAGBuilder::visitZExt(const User &I) {
3523 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3524 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3525 SDValue N = getValue(I.getOperand(0));
3527 I.getType());
3529}
3530
3531void SelectionDAGBuilder::visitSExt(const User &I) {
3532 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3533 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3534 SDValue N = getValue(I.getOperand(0));
3536 I.getType());
3538}
3539
3540void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3541 // FPTrunc is never a no-op cast, no need to check
3542 SDValue N = getValue(I.getOperand(0));
3543 SDLoc dl = getCurSDLoc();
3545 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3546 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3548 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3549}
3550
3551void SelectionDAGBuilder::visitFPExt(const User &I) {
3552 // FPExt is never a no-op cast, no need to check
3553 SDValue N = getValue(I.getOperand(0));
3555 I.getType());
3557}
3558
3559void SelectionDAGBuilder::visitFPToUI(const User &I) {
3560 // FPToUI is never a no-op cast, no need to check
3561 SDValue N = getValue(I.getOperand(0));
3563 I.getType());
3565}
3566
3567void SelectionDAGBuilder::visitFPToSI(const User &I) {
3568 // FPToSI is never a no-op cast, no need to check
3569 SDValue N = getValue(I.getOperand(0));
3571 I.getType());
3573}
3574
3575void SelectionDAGBuilder::visitUIToFP(const User &I) {
3576 // UIToFP is never a no-op cast, no need to check
3577 SDValue N = getValue(I.getOperand(0));
3579 I.getType());
3581}
3582
3583void SelectionDAGBuilder::visitSIToFP(const User &I) {
3584 // SIToFP is never a no-op cast, no need to check
3585 SDValue N = getValue(I.getOperand(0));
3587 I.getType());
3589}
3590
3591void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3592 // What to do depends on the size of the integer and the size of the pointer.
3593 // We can either truncate, zero extend, or no-op, accordingly.
3594 SDValue N = getValue(I.getOperand(0));
3595 auto &TLI = DAG.getTargetLoweringInfo();
3597 I.getType());
3598 EVT PtrMemVT =
3599 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3600 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3601 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3602 setValue(&I, N);
3603}
3604
3605void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3606 // What to do depends on the size of the integer and the size of the pointer.
3607 // We can either truncate, zero extend, or no-op, accordingly.
3608 SDValue N = getValue(I.getOperand(0));
3609 auto &TLI = DAG.getTargetLoweringInfo();
3610 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3611 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3612 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3613 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3614 setValue(&I, N);
3615}
3616
3617void SelectionDAGBuilder::visitBitCast(const User &I) {
3618 SDValue N = getValue(I.getOperand(0));
3619 SDLoc dl = getCurSDLoc();
3621 I.getType());
3622
3623 // BitCast assures us that source and destination are the same size so this is
3624 // either a BITCAST or a no-op.
3625 if (DestVT != N.getValueType())
3627 DestVT, N)); // convert types.
3628 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3629 // might fold any kind of constant expression to an integer constant and that
3630 // is not what we are looking for. Only recognize a bitcast of a genuine
3631 // constant integer as an opaque constant.
3632 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3633 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3634 /*isOpaque*/true));
3635 else
3636 setValue(&I, N); // noop cast.
3637}
3638
3639void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3641 const Value *SV = I.getOperand(0);
3642 SDValue N = getValue(SV);
3643 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3644
3645 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3646 unsigned DestAS = I.getType()->getPointerAddressSpace();
3647
3648 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3649 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3650
3651 setValue(&I, N);
3652}
3653
3654void SelectionDAGBuilder::visitInsertElement(const User &I) {
3656 SDValue InVec = getValue(I.getOperand(0));
3657 SDValue InVal = getValue(I.getOperand(1));
3658 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3661 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3662 InVec, InVal, InIdx));
3663}
3664
3665void SelectionDAGBuilder::visitExtractElement(const User &I) {
3667 SDValue InVec = getValue(I.getOperand(0));
3668 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3671 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3672 InVec, InIdx));
3673}
3674
3675void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3676 SDValue Src1 = getValue(I.getOperand(0));
3677 SDValue Src2 = getValue(I.getOperand(1));
3679 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3680 Mask = SVI->getShuffleMask();
3681 else
3682 Mask = cast<ConstantExpr>(I).getShuffleMask();
3683 SDLoc DL = getCurSDLoc();
3685 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3686 EVT SrcVT = Src1.getValueType();
3687
3688 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3689 VT.isScalableVector()) {
3690 // Canonical splat form of first element of first input vector.
3691 SDValue FirstElt =
3694 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3695 return;
3696 }
3697
3698 // For now, we only handle splats for scalable vectors.
3699 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3700 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3701 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3702
3703 unsigned SrcNumElts = SrcVT.getVectorNumElements();
3704 unsigned MaskNumElts = Mask.size();
3705
3706 if (SrcNumElts == MaskNumElts) {
3707 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3708 return;
3709 }
3710
3711 // Normalize the shuffle vector since mask and vector length don't match.
3712 if (SrcNumElts < MaskNumElts) {
3713 // Mask is longer than the source vectors. We can use concatenate vector to
3714 // make the mask and vectors lengths match.
3715
3716 if (MaskNumElts % SrcNumElts == 0) {
3717 // Mask length is a multiple of the source vector length.
3718 // Check if the shuffle is some kind of concatenation of the input
3719 // vectors.
3720 unsigned NumConcat = MaskNumElts / SrcNumElts;
3721 bool IsConcat = true;
3722 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3723 for (unsigned i = 0; i != MaskNumElts; ++i) {
3724 int Idx = Mask[i];
3725 if (Idx < 0)
3726 continue;
3727 // Ensure the indices in each SrcVT sized piece are sequential and that
3728 // the same source is used for the whole piece.
3729 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3730 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3731 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3732 IsConcat = false;
3733 break;
3734 }
3735 // Remember which source this index came from.
3736 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3737 }
3738
3739 // The shuffle is concatenating multiple vectors together. Just emit
3740 // a CONCAT_VECTORS operation.
3741 if (IsConcat) {
3742 SmallVector<SDValue, 8> ConcatOps;
3743 for (auto Src : ConcatSrcs) {
3744 if (Src < 0)
3745 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3746 else if (Src == 0)
3747 ConcatOps.push_back(Src1);
3748 else
3749 ConcatOps.push_back(Src2);
3750 }
3751 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3752 return;
3753 }
3754 }
3755
3756 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3757 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3758 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3759 PaddedMaskNumElts);
3760
3761 // Pad both vectors with undefs to make them the same length as the mask.
3762 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3763
3764 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3765 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3766 MOps1[0] = Src1;
3767 MOps2[0] = Src2;
3768
3769 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3770 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3771
3772 // Readjust mask for new input vector length.
3773 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3774 for (unsigned i = 0; i != MaskNumElts; ++i) {
3775 int Idx = Mask[i];
3776 if (Idx >= (int)SrcNumElts)
3777 Idx -= SrcNumElts - PaddedMaskNumElts;
3778 MappedOps[i] = Idx;
3779 }
3780
3781 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3782
3783 // If the concatenated vector was padded, extract a subvector with the
3784 // correct number of elements.
3785 if (MaskNumElts != PaddedMaskNumElts)
3788
3789 setValue(&I, Result);
3790 return;
3791 }
3792
3793 if (SrcNumElts > MaskNumElts) {
3794 // Analyze the access pattern of the vector to see if we can extract
3795 // two subvectors and do the shuffle.
3796 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3797 bool CanExtract = true;
3798 for (int Idx : Mask) {
3799 unsigned Input = 0;
3800 if (Idx < 0)
3801 continue;
3802
3803 if (Idx >= (int)SrcNumElts) {
3804 Input = 1;
3805 Idx -= SrcNumElts;
3806 }
3807
3808 // If all the indices come from the same MaskNumElts sized portion of
3809 // the sources we can use extract. Also make sure the extract wouldn't
3810 // extract past the end of the source.
3811 int NewStartIdx = alignDown(Idx, MaskNumElts);
3812 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3813 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3814 CanExtract = false;
3815 // Make sure we always update StartIdx as we use it to track if all
3816 // elements are undef.
3817 StartIdx[Input] = NewStartIdx;
3818 }
3819
3820 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3821 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3822 return;
3823 }
3824 if (CanExtract) {
3825 // Extract appropriate subvector and generate a vector shuffle
3826 for (unsigned Input = 0; Input < 2; ++Input) {
3827 SDValue &Src = Input == 0 ? Src1 : Src2;
3828 if (StartIdx[Input] < 0)
3829 Src = DAG.getUNDEF(VT);
3830 else {
3831 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3832 DAG.getVectorIdxConstant(StartIdx[Input], DL));
3833 }
3834 }
3835
3836 // Calculate new mask.
3837 SmallVector<int, 8> MappedOps(Mask);
3838 for (int &Idx : MappedOps) {
3839 if (Idx >= (int)SrcNumElts)
3840 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3841 else if (Idx >= 0)
3842 Idx -= StartIdx[0];
3843 }
3844
3845 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3846 return;
3847 }
3848 }
3849
3850 // We can't use either concat vectors or extract subvectors so fall back to
3851 // replacing the shuffle with extract and build vector.
3852 // to insert and build vector.
3853 EVT EltVT = VT.getVectorElementType();
3855 for (int Idx : Mask) {
3856 SDValue Res;
3857
3858 if (Idx < 0) {
3859 Res = DAG.getUNDEF(EltVT);
3860 } else {
3861 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3862 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3863
3864 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3866 }
3867
3868 Ops.push_back(Res);
3869 }
3870
3871 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3872}
3873
3874void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3875 ArrayRef<unsigned> Indices = I.getIndices();
3876 const Value *Op0 = I.getOperand(0);
3877 const Value *Op1 = I.getOperand(1);
3878 Type *AggTy = I.getType();
3879 Type *ValTy = Op1->getType();
3880 bool IntoUndef = isa<UndefValue>(Op0);
3881 bool FromUndef = isa<UndefValue>(Op1);
3882
3883 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3884
3886 SmallVector<EVT, 4> AggValueVTs;
3887 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3888 SmallVector<EVT, 4> ValValueVTs;
3889 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3890
3891 unsigned NumAggValues = AggValueVTs.size();
3892 unsigned NumValValues = ValValueVTs.size();
3893 SmallVector<SDValue, 4> Values(NumAggValues);
3894
3895 // Ignore an insertvalue that produces an empty object
3896 if (!NumAggValues) {
3897 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3898 return;
3899 }
3900
3901 SDValue Agg = getValue(Op0);
3902 unsigned i = 0;
3903 // Copy the beginning value(s) from the original aggregate.
3904 for (; i != LinearIndex; ++i)
3905 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3906 SDValue(Agg.getNode(), Agg.getResNo() + i);
3907 // Copy values from the inserted value(s).
3908 if (NumValValues) {
3909 SDValue Val = getValue(Op1);
3910 for (; i != LinearIndex + NumValValues; ++i)
3911 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3912 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3913 }
3914 // Copy remaining value(s) from the original aggregate.
3915 for (; i != NumAggValues; ++i)
3916 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3917 SDValue(Agg.getNode(), Agg.getResNo() + i);
3918
3920 DAG.getVTList(AggValueVTs), Values));
3921}
3922
3923void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3924 ArrayRef<unsigned> Indices = I.getIndices();
3925 const Value *Op0 = I.getOperand(0);
3926 Type *AggTy = Op0->getType();
3927 Type *ValTy = I.getType();
3928 bool OutOfUndef = isa<UndefValue>(Op0);
3929
3930 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3931
3933 SmallVector<EVT, 4> ValValueVTs;
3934 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3935
3936 unsigned NumValValues = ValValueVTs.size();
3937
3938 // Ignore a extractvalue that produces an empty object
3939 if (!NumValValues) {
3940 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3941 return;
3942 }
3943
3944 SmallVector<SDValue, 4> Values(NumValValues);
3945
3946 SDValue Agg = getValue(Op0);
3947 // Copy out the selected value(s).
3948 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3949 Values[i - LinearIndex] =
3950 OutOfUndef ?
3951 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3952 SDValue(Agg.getNode(), Agg.getResNo() + i);
3953
3955 DAG.getVTList(ValValueVTs), Values));
3956}
3957
3958void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3959 Value *Op0 = I.getOperand(0);
3960 // Note that the pointer operand may be a vector of pointers. Take the scalar
3961 // element which holds a pointer.
3962 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3963 SDValue N = getValue(Op0);
3964 SDLoc dl = getCurSDLoc();
3965 auto &TLI = DAG.getTargetLoweringInfo();
3966
3967 // Normalize Vector GEP - all scalar operands should be converted to the
3968 // splat vector.
3969 bool IsVectorGEP = I.getType()->isVectorTy();
3970 ElementCount VectorElementCount =
3971 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3973
3974 if (IsVectorGEP && !N.getValueType().isVector()) {
3976 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3977 N = DAG.getSplat(VT, dl, N);
3978 }
3979
3981 GTI != E; ++GTI) {
3982 const Value *Idx = GTI.getOperand();
3983 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3984 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3985 if (Field) {
3986 // N = N + Offset
3989
3990 // In an inbounds GEP with an offset that is nonnegative even when
3991 // interpreted as signed, assume there is no unsigned overflow.
3993 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3994 Flags.setNoUnsignedWrap(true);
3995
3996 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3997 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3998 }
3999 } else {
4000 // IdxSize is the width of the arithmetic according to IR semantics.
4001 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4002 // (and fix up the result later).
4003 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4004 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4005 TypeSize ElementSize =
4006 DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
4007 // We intentionally mask away the high bits here; ElementSize may not
4008 // fit in IdxTy.
4009 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
4010 bool ElementScalable = ElementSize.isScalable();
4011
4012 // If this is a scalar constant or a splat vector of constants,
4013 // handle it quickly.
4014 const auto *C = dyn_cast<Constant>(Idx);
4015 if (C && isa<VectorType>(C->getType()))
4016 C = C->getSplatValue();
4017
4018 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4019 if (CI && CI->isZero())
4020 continue;
4021 if (CI && !ElementScalable) {
4022 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4024 SDValue OffsVal;
4025 if (IsVectorGEP)
4026 OffsVal = DAG.getConstant(
4027 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4028 else
4029 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4030
4031 // In an inbounds GEP with an offset that is nonnegative even when
4032 // interpreted as signed, assume there is no unsigned overflow.
4034 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
4035 Flags.setNoUnsignedWrap(true);
4036
4037 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4038
4039 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
4040 continue;
4041 }
4042
4043 // N = N + Idx * ElementMul;
4044 SDValue IdxN = getValue(Idx);
4045
4046 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4048 VectorElementCount);
4049 IdxN = DAG.getSplat(VT, dl, IdxN);
4050 }
4051
4052 // If the index is smaller or larger than intptr_t, truncate or extend
4053 // it.
4054 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4055
4056 if (ElementScalable) {
4057 EVT VScaleTy = N.getValueType().getScalarType();
4058 SDValue VScale = DAG.getNode(
4059 ISD::VSCALE, dl, VScaleTy,
4060 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4061 if (IsVectorGEP)
4062 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4063 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
4064 } else {
4065 // If this is a multiply by a power of two, turn it into a shl
4066 // immediately. This is a very common case.
4067 if (ElementMul != 1) {
4068 if (ElementMul.isPowerOf2()) {
4069 unsigned Amt = ElementMul.logBase2();
4070 IdxN = DAG.getNode(ISD::SHL, dl,
4071 N.getValueType(), IdxN,
4072 DAG.getConstant(Amt, dl, IdxN.getValueType()));
4073 } else {
4074 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4075 IdxN.getValueType());
4076 IdxN = DAG.getNode(ISD::MUL, dl,
4077 N.getValueType(), IdxN, Scale);
4078 }
4079 }
4080 }
4081
4082 N = DAG.getNode(ISD::ADD, dl,
4083 N.getValueType(), N, IdxN);
4084 }
4085 }
4086
4087 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4088 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4089 if (IsVectorGEP) {
4090 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4091 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4092 }
4093
4094 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4095 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4096
4097 setValue(&I, N);
4098}
4099
4100void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4101 // If this is a fixed sized alloca in the entry block of the function,
4102 // allocate it statically on the stack.
4103 if (FuncInfo.StaticAllocaMap.count(&I))
4104 return; // getValue will auto-populate this.
4105
4106 SDLoc dl = getCurSDLoc();
4107 Type *Ty = I.getAllocatedType();
4109 auto &DL = DAG.getDataLayout();
4110 TypeSize TySize = DL.getTypeAllocSize(Ty);
4111 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4112
4113 SDValue AllocSize = getValue(I.getArraySize());
4114
4115 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), I.getAddressSpace());
4116 if (AllocSize.getValueType() != IntPtr)
4117 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4118
4119 if (TySize.isScalable())
4120 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4121 DAG.getVScale(dl, IntPtr,
4122 APInt(IntPtr.getScalarSizeInBits(),
4123 TySize.getKnownMinValue())));
4124 else
4125 AllocSize =
4126 DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4127 DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));
4128
4129 // Handle alignment. If the requested alignment is less than or equal to
4130 // the stack alignment, ignore it. If the size is greater than or equal to
4131 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4133 if (*Alignment <= StackAlign)
4134 Alignment = std::nullopt;
4135
4136 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4137 // Round the size of the allocation up to the stack alignment size
4138 // by add SA-1 to the size. This doesn't overflow because we're computing
4139 // an address inside an alloca.
4141 Flags.setNoUnsignedWrap(true);
4142 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4143 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4144
4145 // Mask out the low bits for alignment purposes.
4146 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4147 DAG.getConstant(~StackAlignMask, dl, IntPtr));
4148
4149 SDValue Ops[] = {
4150 getRoot(), AllocSize,
4151 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4152 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4153 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4154 setValue(&I, DSA);
4155 DAG.setRoot(DSA.getValue(1));
4156
4158}
4159
4160static const MDNode *getRangeMetadata(const Instruction &I) {
4161 // If !noundef is not present, then !range violation results in a poison
4162 // value rather than immediate undefined behavior. In theory, transferring
4163 // these annotations to SDAG is fine, but in practice there are key SDAG
4164 // transforms that are known not to be poison-safe, such as folding logical
4165 // and/or to bitwise and/or. For now, only transfer !range if !noundef is
4166 // also present.
4167 if (!I.hasMetadata(LLVMContext::MD_noundef))
4168 return nullptr;
4169 return I.getMetadata(LLVMContext::MD_range);
4170}
4171
4172void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4173 if (I.isAtomic())
4174 return visitAtomicLoad(I);