LLVM 19.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
19#include "llvm/ADT/SmallSet.h"
36#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/Metadata.h"
42#include "llvm/IR/Type.h"
45#include "llvm/Support/Debug.h"
51#include <cassert>
52#include <cstdint>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "legalizedag"
59
60namespace {
61
62/// Keeps track of state when getting the sign of a floating-point value as an
63/// integer.
64struct FloatSignAsInt {
65 EVT FloatVT;
66 SDValue Chain;
67 SDValue FloatPtr;
68 SDValue IntPtr;
69 MachinePointerInfo IntPointerInfo;
70 MachinePointerInfo FloatPointerInfo;
71 SDValue IntValue;
72 APInt SignMask;
73 uint8_t SignBit;
74};
75
76//===----------------------------------------------------------------------===//
77/// This takes an arbitrary SelectionDAG as input and
78/// hacks on it until the target machine can handle it. This involves
79/// eliminating value sizes the machine cannot handle (promoting small sizes to
80/// large sizes or splitting up large values into small values) as well as
81/// eliminating operations the machine cannot handle.
82///
83/// This code also does a small amount of optimization and recognition of idioms
84/// as part of its processing. For example, if a target does not support a
85/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
86/// will attempt merge setcc and brc instructions into brcc's.
87class SelectionDAGLegalize {
88 const TargetMachine &TM;
89 const TargetLowering &TLI;
90 SelectionDAG &DAG;
91
92 /// The set of nodes which have already been legalized. We hold a
93 /// reference to it in order to update as necessary on node deletion.
94 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
95
96 /// A set of all the nodes updated during legalization.
97 SmallSetVector<SDNode *, 16> *UpdatedNodes;
98
99 EVT getSetCCResultType(EVT VT) const {
100 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
101 }
102
103 // Libcall insertion helpers.
104
105public:
106 SelectionDAGLegalize(SelectionDAG &DAG,
107 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
108 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112 /// Legalizes the given operation.
113 void LegalizeOp(SDNode *Node);
114
115private:
116 SDValue OptimizeFloatStore(StoreSDNode *ST);
117
118 void LegalizeLoadOps(SDNode *Node);
119 void LegalizeStoreOps(SDNode *Node);
120
121 /// Some targets cannot handle a variable
122 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
123 /// is necessary to spill the vector being inserted into to memory, perform
124 /// the insert there, and then read the result back.
125 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
128 const SDLoc &dl);
129
130 /// Return a vector shuffle operation which
131 /// performs the same shuffe in terms of order or result bytes, but on a type
132 /// whose vector element type is narrower than the original shuffle type.
133 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
134 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
135 SDValue N1, SDValue N2,
136 ArrayRef<int> Mask) const;
137
138 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
140 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
141
142 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
145 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128,
150 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
151 RTLIB::Libcall Call_I8,
152 RTLIB::Libcall Call_I16,
153 RTLIB::Libcall Call_I32,
154 RTLIB::Libcall Call_I64,
155 RTLIB::Libcall Call_I128);
156 void ExpandArgFPLibCall(SDNode *Node,
157 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
158 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
159 RTLIB::Libcall Call_PPCF128,
161 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
163
164 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
165 const SDLoc &dl);
166 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
167 const SDLoc &dl, SDValue ChainIn);
168 SDValue ExpandBUILD_VECTOR(SDNode *Node);
169 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
170 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
171 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
173 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
174 SDValue Value) const;
175 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
176 SDValue NewIntValue) const;
177 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
178 SDValue ExpandFABS(SDNode *Node) const;
179 SDValue ExpandFNEG(SDNode *Node) const;
180 SDValue expandLdexp(SDNode *Node) const;
181 SDValue expandFrexp(SDNode *Node) const;
182
183 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
184 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
186 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
188 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
189
190 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
191
192 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
193 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
194 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
195
196 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
197 SDValue ExpandConstant(ConstantSDNode *CP);
198
199 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
200 bool ExpandNode(SDNode *Node);
201 void ConvertNodeToLibcall(SDNode *Node);
202 void PromoteNode(SDNode *Node);
203
204public:
205 // Node replacement helpers
206
207 void ReplacedNode(SDNode *N) {
208 LegalizedNodes.erase(N);
209 if (UpdatedNodes)
210 UpdatedNodes->insert(N);
211 }
212
213 void ReplaceNode(SDNode *Old, SDNode *New) {
214 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
215 dbgs() << " with: "; New->dump(&DAG));
216
217 assert(Old->getNumValues() == New->getNumValues() &&
218 "Replacing one node with another that produces a different number "
219 "of values!");
220 DAG.ReplaceAllUsesWith(Old, New);
221 if (UpdatedNodes)
222 UpdatedNodes->insert(New);
223 ReplacedNode(Old);
224 }
225
226 void ReplaceNode(SDValue Old, SDValue New) {
227 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
228 dbgs() << " with: "; New->dump(&DAG));
229
230 DAG.ReplaceAllUsesWith(Old, New);
231 if (UpdatedNodes)
232 UpdatedNodes->insert(New.getNode());
233 ReplacedNode(Old.getNode());
234 }
235
236 void ReplaceNode(SDNode *Old, const SDValue *New) {
237 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
238
239 DAG.ReplaceAllUsesWith(Old, New);
240 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
241 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
242 New[i]->dump(&DAG));
243 if (UpdatedNodes)
244 UpdatedNodes->insert(New[i].getNode());
245 }
246 ReplacedNode(Old);
247 }
248
249 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
250 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
251 dbgs() << " with: "; New->dump(&DAG));
252
253 DAG.ReplaceAllUsesOfValueWith(Old, New);
254 if (UpdatedNodes)
255 UpdatedNodes->insert(New.getNode());
256 ReplacedNode(Old.getNode());
257 }
258};
259
260} // end anonymous namespace
261
262// Helper function that generates an MMO that considers the alignment of the
263// stack, and the size of the stack object
265 MachineFunction &MF,
266 bool isObjectScalable) {
267 auto &MFI = MF.getFrameInfo();
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
270 uint64_t ObjectSize = isObjectScalable ? ~UINT64_C(0) : MFI.getObjectSize(FI);
272 ObjectSize, MFI.getObjectAlign(FI));
273}
274
275/// Return a vector shuffle operation which
276/// performs the same shuffle in terms of order or result bytes, but on a type
277/// whose vector element type is narrower than the original shuffle type.
278/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
279SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
280 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
281 ArrayRef<int> Mask) const {
282 unsigned NumMaskElts = VT.getVectorNumElements();
283 unsigned NumDestElts = NVT.getVectorNumElements();
284 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
285
286 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
287
288 if (NumEltsGrowth == 1)
289 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
290
291 SmallVector<int, 8> NewMask;
292 for (unsigned i = 0; i != NumMaskElts; ++i) {
293 int Idx = Mask[i];
294 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
295 if (Idx < 0)
296 NewMask.push_back(-1);
297 else
298 NewMask.push_back(Idx * NumEltsGrowth + j);
299 }
300 }
301 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
302 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
303 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
304}
305
306/// Expands the ConstantFP node to an integer constant or
307/// a load from the constant pool.
309SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
310 bool Extend = false;
311 SDLoc dl(CFP);
312
313 // If a FP immediate is precise when represented as a float and if the
314 // target can do an extending load from float to double, we put it into
315 // the constant pool as a float, even if it's is statically typed as a
316 // double. This shrinks FP constants and canonicalizes them for targets where
317 // an FP extending load is the same cost as a normal load (such as on the x87
318 // fp stack or PPC FP unit).
319 EVT VT = CFP->getValueType(0);
320 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
321 if (!UseCP) {
322 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
323 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
324 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
325 }
326
327 APFloat APF = CFP->getValueAPF();
328 EVT OrigVT = VT;
329 EVT SVT = VT;
330
331 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
332 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
333 if (!APF.isSignaling()) {
334 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
335 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
337 // Only do this if the target has a native EXTLOAD instruction from
338 // smaller type.
339 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
340 TLI.ShouldShrinkFPConstant(OrigVT)) {
341 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
342 LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(
343 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
344 VT = SVT;
345 Extend = true;
346 }
347 }
348 }
349
350 SDValue CPIdx =
351 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
352 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
353 if (Extend) {
354 SDValue Result = DAG.getExtLoad(
355 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
356 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
357 Alignment);
358 return Result;
359 }
360 SDValue Result = DAG.getLoad(
361 OrigVT, dl, DAG.getEntryNode(), CPIdx,
362 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
363 return Result;
364}
365
366/// Expands the Constant node to a load from the constant pool.
367SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
368 SDLoc dl(CP);
369 EVT VT = CP->getValueType(0);
370 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
371 TLI.getPointerTy(DAG.getDataLayout()));
372 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
373 SDValue Result = DAG.getLoad(
374 VT, dl, DAG.getEntryNode(), CPIdx,
375 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
376 return Result;
377}
378
379/// Some target cannot handle a variable insertion index for the
380/// INSERT_VECTOR_ELT instruction. In this case, it
381/// is necessary to spill the vector being inserted into to memory, perform
382/// the insert there, and then read the result back.
383SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
384 SDValue Val,
385 SDValue Idx,
386 const SDLoc &dl) {
387 SDValue Tmp1 = Vec;
388 SDValue Tmp2 = Val;
389 SDValue Tmp3 = Idx;
390
391 // If the target doesn't support this, we have to spill the input vector
392 // to a temporary stack slot, update the element, then reload it. This is
393 // badness. We could also load the value into a vector register (either
394 // with a "move to register" or "extload into register" instruction, then
395 // permute it into place, if the idx is a constant and if the idx is
396 // supported by the target.
397 EVT VT = Tmp1.getValueType();
398 EVT EltVT = VT.getVectorElementType();
399 SDValue StackPtr = DAG.CreateStackTemporary(VT);
400
401 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
402
403 // Store the vector.
404 SDValue Ch = DAG.getStore(
405 DAG.getEntryNode(), dl, Tmp1, StackPtr,
406 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
407
408 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
409
410 // Store the scalar value.
411 Ch = DAG.getTruncStore(
412 Ch, dl, Tmp2, StackPtr2,
413 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
414 // Load the updated vector.
415 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
416 DAG.getMachineFunction(), SPFI));
417}
418
419SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
420 SDValue Idx,
421 const SDLoc &dl) {
422 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
423 // SCALAR_TO_VECTOR requires that the type of the value being inserted
424 // match the element type of the vector being created, except for
425 // integers in which case the inserted value can be over width.
426 EVT EltVT = Vec.getValueType().getVectorElementType();
427 if (Val.getValueType() == EltVT ||
428 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
429 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
430 Vec.getValueType(), Val);
431
432 unsigned NumElts = Vec.getValueType().getVectorNumElements();
433 // We generate a shuffle of InVec and ScVec, so the shuffle mask
434 // should be 0,1,2,3,4,5... with the appropriate element replaced with
435 // elt 0 of the RHS.
436 SmallVector<int, 8> ShufOps;
437 for (unsigned i = 0; i != NumElts; ++i)
438 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
439
440 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
441 }
442 }
443 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
444}
445
446SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
447 if (!ISD::isNormalStore(ST))
448 return SDValue();
449
450 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
451 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
452 // FIXME: move this to the DAG Combiner! Note that we can't regress due
453 // to phase ordering between legalized code and the dag combiner. This
454 // probably means that we need to integrate dag combiner and legalizer
455 // together.
456 // We generally can't do this one for long doubles.
457 SDValue Chain = ST->getChain();
458 SDValue Ptr = ST->getBasePtr();
459 SDValue Value = ST->getValue();
460 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
461 AAMDNodes AAInfo = ST->getAAInfo();
462 SDLoc dl(ST);
463
464 // Don't optimise TargetConstantFP
465 if (Value.getOpcode() == ISD::TargetConstantFP)
466 return SDValue();
467
468 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
469 if (CFP->getValueType(0) == MVT::f32 &&
470 TLI.isTypeLegal(MVT::i32)) {
471 SDValue Con = DAG.getConstant(CFP->getValueAPF().
472 bitcastToAPInt().zextOrTrunc(32),
473 SDLoc(CFP), MVT::i32);
474 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
475 ST->getOriginalAlign(), MMOFlags, AAInfo);
476 }
477
478 if (CFP->getValueType(0) == MVT::f64 &&
479 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
480 // If this target supports 64-bit registers, do a single 64-bit store.
481 if (TLI.isTypeLegal(MVT::i64)) {
482 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
483 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
484 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
485 ST->getOriginalAlign(), MMOFlags, AAInfo);
486 }
487
488 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
489 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
490 // stores. If the target supports neither 32- nor 64-bits, this
491 // xform is certainly not worth it.
492 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
493 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
494 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
495 if (DAG.getDataLayout().isBigEndian())
496 std::swap(Lo, Hi);
497
498 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
499 ST->getOriginalAlign(), MMOFlags, AAInfo);
500 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl);
501 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
502 ST->getPointerInfo().getWithOffset(4),
503 ST->getOriginalAlign(), MMOFlags, AAInfo);
504
505 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
506 }
507 }
508 }
509 return SDValue();
510}
511
512void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
513 StoreSDNode *ST = cast<StoreSDNode>(Node);
514 SDValue Chain = ST->getChain();
515 SDValue Ptr = ST->getBasePtr();
516 SDLoc dl(Node);
517
518 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
519 AAMDNodes AAInfo = ST->getAAInfo();
520
521 if (!ST->isTruncatingStore()) {
522 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
523 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
524 ReplaceNode(ST, OptStore);
525 return;
526 }
527
528 SDValue Value = ST->getValue();
529 MVT VT = Value.getSimpleValueType();
530 switch (TLI.getOperationAction(ISD::STORE, VT)) {
531 default: llvm_unreachable("This action is not supported yet!");
532 case TargetLowering::Legal: {
533 // If this is an unaligned store and the target doesn't support it,
534 // expand it.
535 EVT MemVT = ST->getMemoryVT();
536 const DataLayout &DL = DAG.getDataLayout();
537 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
538 *ST->getMemOperand())) {
539 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
540 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
541 ReplaceNode(SDValue(ST, 0), Result);
542 } else
543 LLVM_DEBUG(dbgs() << "Legal store\n");
544 break;
545 }
546 case TargetLowering::Custom: {
547 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
548 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
549 if (Res && Res != SDValue(Node, 0))
550 ReplaceNode(SDValue(Node, 0), Res);
551 return;
552 }
553 case TargetLowering::Promote: {
554 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
555 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
556 "Can only promote stores to same size type");
557 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
558 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
559 ST->getOriginalAlign(), MMOFlags, AAInfo);
560 ReplaceNode(SDValue(Node, 0), Result);
561 break;
562 }
563 }
564 return;
565 }
566
567 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
568 SDValue Value = ST->getValue();
569 EVT StVT = ST->getMemoryVT();
570 TypeSize StWidth = StVT.getSizeInBits();
571 TypeSize StSize = StVT.getStoreSizeInBits();
572 auto &DL = DAG.getDataLayout();
573
574 if (StWidth != StSize) {
575 // Promote to a byte-sized store with upper bits zero if not
576 // storing an integral number of bytes. For example, promote
577 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
578 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue());
579 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
581 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
582 ST->getOriginalAlign(), MMOFlags, AAInfo);
583 ReplaceNode(SDValue(Node, 0), Result);
584 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedValue())) {
585 // If not storing a power-of-2 number of bits, expand as two stores.
586 assert(!StVT.isVector() && "Unsupported truncstore!");
587 unsigned StWidthBits = StWidth.getFixedValue();
588 unsigned LogStWidth = Log2_32(StWidthBits);
589 assert(LogStWidth < 32);
590 unsigned RoundWidth = 1 << LogStWidth;
591 assert(RoundWidth < StWidthBits);
592 unsigned ExtraWidth = StWidthBits - RoundWidth;
593 assert(ExtraWidth < RoundWidth);
594 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
595 "Store size not an integral number of bytes!");
596 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
597 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
598 SDValue Lo, Hi;
599 unsigned IncrementSize;
600
601 if (DL.isLittleEndian()) {
602 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
603 // Store the bottom RoundWidth bits.
604 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
605 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
606
607 // Store the remaining ExtraWidth bits.
608 IncrementSize = RoundWidth / 8;
609 Ptr =
610 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
611 Hi = DAG.getNode(
612 ISD::SRL, dl, Value.getValueType(), Value,
613 DAG.getConstant(RoundWidth, dl,
614 TLI.getShiftAmountTy(Value.getValueType(), DL)));
615 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
616 ST->getPointerInfo().getWithOffset(IncrementSize),
617 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
618 } else {
619 // Big endian - avoid unaligned stores.
620 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
621 // Store the top RoundWidth bits.
622 Hi = DAG.getNode(
623 ISD::SRL, dl, Value.getValueType(), Value,
624 DAG.getConstant(ExtraWidth, dl,
625 TLI.getShiftAmountTy(Value.getValueType(), DL)));
626 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
627 ST->getOriginalAlign(), MMOFlags, AAInfo);
628
629 // Store the remaining ExtraWidth bits.
630 IncrementSize = RoundWidth / 8;
631 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
632 DAG.getConstant(IncrementSize, dl,
633 Ptr.getValueType()));
634 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
635 ST->getPointerInfo().getWithOffset(IncrementSize),
636 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
637 }
638
639 // The order of the stores doesn't matter.
640 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
641 ReplaceNode(SDValue(Node, 0), Result);
642 } else {
643 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
644 default: llvm_unreachable("This action is not supported yet!");
645 case TargetLowering::Legal: {
646 EVT MemVT = ST->getMemoryVT();
647 // If this is an unaligned store and the target doesn't support it,
648 // expand it.
649 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
650 *ST->getMemOperand())) {
651 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
652 ReplaceNode(SDValue(ST, 0), Result);
653 }
654 break;
655 }
656 case TargetLowering::Custom: {
657 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
658 if (Res && Res != SDValue(Node, 0))
659 ReplaceNode(SDValue(Node, 0), Res);
660 return;
661 }
662 case TargetLowering::Expand:
663 assert(!StVT.isVector() &&
664 "Vector Stores are handled in LegalizeVectorOps");
665
667
668 // TRUNCSTORE:i16 i32 -> STORE i16
669 if (TLI.isTypeLegal(StVT)) {
670 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
671 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
672 ST->getOriginalAlign(), MMOFlags, AAInfo);
673 } else {
674 // The in-memory type isn't legal. Truncate to the type it would promote
675 // to, and then do a truncstore.
676 Value = DAG.getNode(ISD::TRUNCATE, dl,
677 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
678 Value);
679 Result =
680 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
681 ST->getOriginalAlign(), MMOFlags, AAInfo);
682 }
683
684 ReplaceNode(SDValue(Node, 0), Result);
685 break;
686 }
687 }
688}
689
690void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
691 LoadSDNode *LD = cast<LoadSDNode>(Node);
692 SDValue Chain = LD->getChain(); // The chain.
693 SDValue Ptr = LD->getBasePtr(); // The base pointer.
694 SDValue Value; // The value returned by the load op.
695 SDLoc dl(Node);
696
697 ISD::LoadExtType ExtType = LD->getExtensionType();
698 if (ExtType == ISD::NON_EXTLOAD) {
699 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
700 MVT VT = Node->getSimpleValueType(0);
701 SDValue RVal = SDValue(Node, 0);
702 SDValue RChain = SDValue(Node, 1);
703
704 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
705 default: llvm_unreachable("This action is not supported yet!");
706 case TargetLowering::Legal: {
707 EVT MemVT = LD->getMemoryVT();
708 const DataLayout &DL = DAG.getDataLayout();
709 // If this is an unaligned load and the target doesn't support it,
710 // expand it.
711 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
712 *LD->getMemOperand())) {
713 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
714 }
715 break;
716 }
717 case TargetLowering::Custom:
718 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
719 RVal = Res;
720 RChain = Res.getValue(1);
721 }
722 break;
723
724 case TargetLowering::Promote: {
725 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
726 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
727 "Can only promote loads to same size type");
728
729 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
730 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
731 RChain = Res.getValue(1);
732 break;
733 }
734 }
735 if (RChain.getNode() != Node) {
736 assert(RVal.getNode() != Node && "Load must be completely replaced");
737 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
738 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
739 if (UpdatedNodes) {
740 UpdatedNodes->insert(RVal.getNode());
741 UpdatedNodes->insert(RChain.getNode());
742 }
743 ReplacedNode(Node);
744 }
745 return;
746 }
747
748 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
749 EVT SrcVT = LD->getMemoryVT();
750 TypeSize SrcWidth = SrcVT.getSizeInBits();
751 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
752 AAMDNodes AAInfo = LD->getAAInfo();
753
754 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
755 // Some targets pretend to have an i1 loading operation, and actually
756 // load an i8. This trick is correct for ZEXTLOAD because the top 7
757 // bits are guaranteed to be zero; it helps the optimizers understand
758 // that these bits are zero. It is also useful for EXTLOAD, since it
759 // tells the optimizers that those bits are undefined. It would be
760 // nice to have an effective generic way of getting these benefits...
761 // Until such a way is found, don't insist on promoting i1 here.
762 (SrcVT != MVT::i1 ||
763 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
764 TargetLowering::Promote)) {
765 // Promote to a byte-sized load if not loading an integral number of
766 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
767 unsigned NewWidth = SrcVT.getStoreSizeInBits();
768 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
769 SDValue Ch;
770
771 // The extra bits are guaranteed to be zero, since we stored them that
772 // way. A zext load from NVT thus automatically gives zext from SrcVT.
773
774 ISD::LoadExtType NewExtType =
776
777 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
778 Chain, Ptr, LD->getPointerInfo(), NVT,
779 LD->getOriginalAlign(), MMOFlags, AAInfo);
780
781 Ch = Result.getValue(1); // The chain.
782
783 if (ExtType == ISD::SEXTLOAD)
784 // Having the top bits zero doesn't help when sign extending.
785 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
786 Result.getValueType(),
787 Result, DAG.getValueType(SrcVT));
788 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
789 // All the top bits are guaranteed to be zero - inform the optimizers.
790 Result = DAG.getNode(ISD::AssertZext, dl,
791 Result.getValueType(), Result,
792 DAG.getValueType(SrcVT));
793
794 Value = Result;
795 Chain = Ch;
796 } else if (!isPowerOf2_64(SrcWidth.getKnownMinValue())) {
797 // If not loading a power-of-2 number of bits, expand as two loads.
798 assert(!SrcVT.isVector() && "Unsupported extload!");
799 unsigned SrcWidthBits = SrcWidth.getFixedValue();
800 unsigned LogSrcWidth = Log2_32(SrcWidthBits);
801 assert(LogSrcWidth < 32);
802 unsigned RoundWidth = 1 << LogSrcWidth;
803 assert(RoundWidth < SrcWidthBits);
804 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
805 assert(ExtraWidth < RoundWidth);
806 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
807 "Load size not an integral number of bytes!");
808 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
809 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
810 SDValue Lo, Hi, Ch;
811 unsigned IncrementSize;
812 auto &DL = DAG.getDataLayout();
813
814 if (DL.isLittleEndian()) {
815 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
816 // Load the bottom RoundWidth bits.
817 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
818 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
819 MMOFlags, AAInfo);
820
821 // Load the remaining ExtraWidth bits.
822 IncrementSize = RoundWidth / 8;
823 Ptr =
824 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
825 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
826 LD->getPointerInfo().getWithOffset(IncrementSize),
827 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
828
829 // Build a factor node to remember that this load is independent of
830 // the other one.
831 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
832 Hi.getValue(1));
833
834 // Move the top bits to the right place.
835 Hi = DAG.getNode(
836 ISD::SHL, dl, Hi.getValueType(), Hi,
837 DAG.getConstant(RoundWidth, dl,
838 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
839
840 // Join the hi and lo parts.
841 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
842 } else {
843 // Big endian - avoid unaligned loads.
844 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
845 // Load the top RoundWidth bits.
846 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
847 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
848 MMOFlags, AAInfo);
849
850 // Load the remaining ExtraWidth bits.
851 IncrementSize = RoundWidth / 8;
852 Ptr =
853 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
854 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
855 LD->getPointerInfo().getWithOffset(IncrementSize),
856 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
857
858 // Build a factor node to remember that this load is independent of
859 // the other one.
860 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
861 Hi.getValue(1));
862
863 // Move the top bits to the right place.
864 Hi = DAG.getNode(
865 ISD::SHL, dl, Hi.getValueType(), Hi,
866 DAG.getConstant(ExtraWidth, dl,
867 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
868
869 // Join the hi and lo parts.
870 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
871 }
872
873 Chain = Ch;
874 } else {
875 bool isCustom = false;
876 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
877 SrcVT.getSimpleVT())) {
878 default: llvm_unreachable("This action is not supported yet!");
879 case TargetLowering::Custom:
880 isCustom = true;
881 [[fallthrough]];
882 case TargetLowering::Legal:
883 Value = SDValue(Node, 0);
884 Chain = SDValue(Node, 1);
885
886 if (isCustom) {
887 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
888 Value = Res;
889 Chain = Res.getValue(1);
890 }
891 } else {
892 // If this is an unaligned load and the target doesn't support it,
893 // expand it.
894 EVT MemVT = LD->getMemoryVT();
895 const DataLayout &DL = DAG.getDataLayout();
896 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
897 *LD->getMemOperand())) {
898 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
899 }
900 }
901 break;
902
903 case TargetLowering::Expand: {
904 EVT DestVT = Node->getValueType(0);
905 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
906 // If the source type is not legal, see if there is a legal extload to
907 // an intermediate type that we can then extend further.
908 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
909 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
910 (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
911 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
912 // If we are loading a legal type, this is a non-extload followed by a
913 // full extend.
914 ISD::LoadExtType MidExtType =
915 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
916
917 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
918 SrcVT, LD->getMemOperand());
919 unsigned ExtendOp =
921 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
922 Chain = Load.getValue(1);
923 break;
924 }
925
926 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
927 // normal undefined upper bits behavior to allow using an in-reg extend
928 // with the illegal FP type, so load as an integer and do the
929 // from-integer conversion.
930 EVT SVT = SrcVT.getScalarType();
931 if (SVT == MVT::f16 || SVT == MVT::bf16) {
932 EVT ISrcVT = SrcVT.changeTypeToInteger();
933 EVT IDestVT = DestVT.changeTypeToInteger();
934 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
935
936 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
937 Ptr, ISrcVT, LD->getMemOperand());
938 Value =
939 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
940 dl, DestVT, Result);
941 Chain = Result.getValue(1);
942 break;
943 }
944 }
945
946 assert(!SrcVT.isVector() &&
947 "Vector Loads are handled in LegalizeVectorOps");
948
949 // FIXME: This does not work for vectors on most targets. Sign-
950 // and zero-extend operations are currently folded into extending
951 // loads, whether they are legal or not, and then we end up here
952 // without any support for legalizing them.
953 assert(ExtType != ISD::EXTLOAD &&
954 "EXTLOAD should always be supported!");
955 // Turn the unsupported load into an EXTLOAD followed by an
956 // explicit zero/sign extend inreg.
957 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
958 Node->getValueType(0),
959 Chain, Ptr, SrcVT,
960 LD->getMemOperand());
961 SDValue ValRes;
962 if (ExtType == ISD::SEXTLOAD)
963 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
964 Result.getValueType(),
965 Result, DAG.getValueType(SrcVT));
966 else
967 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
968 Value = ValRes;
969 Chain = Result.getValue(1);
970 break;
971 }
972 }
973 }
974
975 // Since loads produce two values, make sure to remember that we legalized
976 // both of them.
977 if (Chain.getNode() != Node) {
978 assert(Value.getNode() != Node && "Load must be completely replaced");
979 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
980 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
981 if (UpdatedNodes) {
982 UpdatedNodes->insert(Value.getNode());
983 UpdatedNodes->insert(Chain.getNode());
984 }
985 ReplacedNode(Node);
986 }
987}
988
989/// Return a legal replacement for the given operation, with all legal operands.
990void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
991 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
992
993 // Allow illegal target nodes and illegal registers.
994 if (Node->getOpcode() == ISD::TargetConstant ||
995 Node->getOpcode() == ISD::Register)
996 return;
997
998#ifndef NDEBUG
999 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1000 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
1001 TargetLowering::TypeLegal &&
1002 "Unexpected illegal type!");
1003
1004 for (const SDValue &Op : Node->op_values())
1005 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
1006 TargetLowering::TypeLegal ||
1007 Op.getOpcode() == ISD::TargetConstant ||
1008 Op.getOpcode() == ISD::Register) &&
1009 "Unexpected illegal type!");
1010#endif
1011
1012 // Figure out the correct action; the way to query this varies by opcode
1013 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1014 bool SimpleFinishLegalizing = true;
1015 switch (Node->getOpcode()) {
1019 case ISD::STACKSAVE:
1020 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1021 break;
1023 Action = TLI.getOperationAction(Node->getOpcode(),
1024 Node->getValueType(0));
1025 break;
1026 case ISD::VAARG:
1027 Action = TLI.getOperationAction(Node->getOpcode(),
1028 Node->getValueType(0));
1029 if (Action != TargetLowering::Promote)
1030 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1031 break;
1032 case ISD::SET_FPENV:
1033 case ISD::SET_FPMODE:
1034 Action = TLI.getOperationAction(Node->getOpcode(),
1035 Node->getOperand(1).getValueType());
1036 break;
1037 case ISD::FP_TO_FP16:
1038 case ISD::FP_TO_BF16:
1039 case ISD::SINT_TO_FP:
1040 case ISD::UINT_TO_FP:
1042 case ISD::LROUND:
1043 case ISD::LLROUND:
1044 case ISD::LRINT:
1045 case ISD::LLRINT:
1046 Action = TLI.getOperationAction(Node->getOpcode(),
1047 Node->getOperand(0).getValueType());
1048 break;
1052 case ISD::STRICT_LRINT:
1053 case ISD::STRICT_LLRINT:
1054 case ISD::STRICT_LROUND:
1056 // These pseudo-ops are the same as the other STRICT_ ops except
1057 // they are registered with setOperationAction() using the input type
1058 // instead of the output type.
1059 Action = TLI.getOperationAction(Node->getOpcode(),
1060 Node->getOperand(1).getValueType());
1061 break;
1063 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1064 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1065 break;
1066 }
1067 case ISD::ATOMIC_STORE:
1068 Action = TLI.getOperationAction(Node->getOpcode(),
1069 Node->getOperand(1).getValueType());
1070 break;
1071 case ISD::SELECT_CC:
1072 case ISD::STRICT_FSETCC:
1074 case ISD::SETCC:
1075 case ISD::SETCCCARRY:
1076 case ISD::VP_SETCC:
1077 case ISD::BR_CC: {
1078 unsigned Opc = Node->getOpcode();
1079 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1080 : Opc == ISD::STRICT_FSETCC ? 3
1081 : Opc == ISD::STRICT_FSETCCS ? 3
1082 : Opc == ISD::SETCCCARRY ? 3
1083 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1084 : 1;
1085 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1086 : Opc == ISD::STRICT_FSETCC ? 1
1087 : Opc == ISD::STRICT_FSETCCS ? 1
1088 : 0;
1089 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1090 ISD::CondCode CCCode =
1091 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1092 Action = TLI.getCondCodeAction(CCCode, OpVT);
1093 if (Action == TargetLowering::Legal) {
1094 if (Node->getOpcode() == ISD::SELECT_CC)
1095 Action = TLI.getOperationAction(Node->getOpcode(),
1096 Node->getValueType(0));
1097 else
1098 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1099 }
1100 break;
1101 }
1102 case ISD::LOAD:
1103 case ISD::STORE:
1104 // FIXME: Model these properly. LOAD and STORE are complicated, and
1105 // STORE expects the unlegalized operand in some cases.
1106 SimpleFinishLegalizing = false;
1107 break;
1108 case ISD::CALLSEQ_START:
1109 case ISD::CALLSEQ_END:
1110 // FIXME: This shouldn't be necessary. These nodes have special properties
1111 // dealing with the recursive nature of legalization. Removing this
1112 // special case should be done as part of making LegalizeDAG non-recursive.
1113 SimpleFinishLegalizing = false;
1114 break;
1116 case ISD::GET_ROUNDING:
1117 case ISD::MERGE_VALUES:
1118 case ISD::EH_RETURN:
1120 case ISD::EH_DWARF_CFA:
1124 // These operations lie about being legal: when they claim to be legal,
1125 // they should actually be expanded.
1126 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1127 if (Action == TargetLowering::Legal)
1128 Action = TargetLowering::Expand;
1129 break;
1132 case ISD::FRAMEADDR:
1133 case ISD::RETURNADDR:
1135 case ISD::SPONENTRY:
1136 // These operations lie about being legal: when they claim to be legal,
1137 // they should actually be custom-lowered.
1138 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1139 if (Action == TargetLowering::Legal)
1140 Action = TargetLowering::Custom;
1141 break;
1144 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1145 // legalization might have expanded that to several smaller types.
1146 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1147 break;
1148 case ISD::READ_REGISTER:
1150 // Named register is legal in the DAG, but blocked by register name
1151 // selection if not implemented by target (to chose the correct register)
1152 // They'll be converted to Copy(To/From)Reg.
1153 Action = TargetLowering::Legal;
1154 break;
1155 case ISD::UBSANTRAP:
1156 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1157 if (Action == TargetLowering::Expand) {
1158 // replace ISD::UBSANTRAP with ISD::TRAP
1159 SDValue NewVal;
1160 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1161 Node->getOperand(0));
1162 ReplaceNode(Node, NewVal.getNode());
1163 LegalizeOp(NewVal.getNode());
1164 return;
1165 }
1166 break;
1167 case ISD::DEBUGTRAP:
1168 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1169 if (Action == TargetLowering::Expand) {
1170 // replace ISD::DEBUGTRAP with ISD::TRAP
1171 SDValue NewVal;
1172 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1173 Node->getOperand(0));
1174 ReplaceNode(Node, NewVal.getNode());
1175 LegalizeOp(NewVal.getNode());
1176 return;
1177 }
1178 break;
1179 case ISD::SADDSAT:
1180 case ISD::UADDSAT:
1181 case ISD::SSUBSAT:
1182 case ISD::USUBSAT:
1183 case ISD::SSHLSAT:
1184 case ISD::USHLSAT:
1187 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1188 break;
1189 case ISD::SMULFIX:
1190 case ISD::SMULFIXSAT:
1191 case ISD::UMULFIX:
1192 case ISD::UMULFIXSAT:
1193 case ISD::SDIVFIX:
1194 case ISD::SDIVFIXSAT:
1195 case ISD::UDIVFIX:
1196 case ISD::UDIVFIXSAT: {
1197 unsigned Scale = Node->getConstantOperandVal(2);
1198 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1199 Node->getValueType(0), Scale);
1200 break;
1201 }
1202 case ISD::MSCATTER:
1203 Action = TLI.getOperationAction(Node->getOpcode(),
1204 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1205 break;
1206 case ISD::MSTORE:
1207 Action = TLI.getOperationAction(Node->getOpcode(),
1208 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1209 break;
1210 case ISD::VP_SCATTER:
1211 Action = TLI.getOperationAction(
1212 Node->getOpcode(),
1213 cast<VPScatterSDNode>(Node)->getValue().getValueType());
1214 break;
1215 case ISD::VP_STORE:
1216 Action = TLI.getOperationAction(
1217 Node->getOpcode(),
1218 cast<VPStoreSDNode>(Node)->getValue().getValueType());
1219 break;
1220 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1221 Action = TLI.getOperationAction(
1222 Node->getOpcode(),
1223 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1224 break;
1227 case ISD::VECREDUCE_ADD:
1228 case ISD::VECREDUCE_MUL:
1229 case ISD::VECREDUCE_AND:
1230 case ISD::VECREDUCE_OR:
1231 case ISD::VECREDUCE_XOR:
1240 case ISD::IS_FPCLASS:
1241 Action = TLI.getOperationAction(
1242 Node->getOpcode(), Node->getOperand(0).getValueType());
1243 break;
1246 case ISD::VP_REDUCE_FADD:
1247 case ISD::VP_REDUCE_FMUL:
1248 case ISD::VP_REDUCE_ADD:
1249 case ISD::VP_REDUCE_MUL:
1250 case ISD::VP_REDUCE_AND:
1251 case ISD::VP_REDUCE_OR:
1252 case ISD::VP_REDUCE_XOR:
1253 case ISD::VP_REDUCE_SMAX:
1254 case ISD::VP_REDUCE_SMIN:
1255 case ISD::VP_REDUCE_UMAX:
1256 case ISD::VP_REDUCE_UMIN:
1257 case ISD::VP_REDUCE_FMAX:
1258 case ISD::VP_REDUCE_FMIN:
1259 case ISD::VP_REDUCE_SEQ_FADD:
1260 case ISD::VP_REDUCE_SEQ_FMUL:
1261 Action = TLI.getOperationAction(
1262 Node->getOpcode(), Node->getOperand(1).getValueType());
1263 break;
1264 default:
1265 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1266 Action = TLI.getCustomOperationAction(*Node);
1267 } else {
1268 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1269 }
1270 break;
1271 }
1272
1273 if (SimpleFinishLegalizing) {
1274 SDNode *NewNode = Node;
1275 switch (Node->getOpcode()) {
1276 default: break;
1277 case ISD::SHL:
1278 case ISD::SRL:
1279 case ISD::SRA:
1280 case ISD::ROTL:
1281 case ISD::ROTR: {
1282 // Legalizing shifts/rotates requires adjusting the shift amount
1283 // to the appropriate width.
1284 SDValue Op0 = Node->getOperand(0);
1285 SDValue Op1 = Node->getOperand(1);
1286 if (!Op1.getValueType().isVector()) {
1287 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1288 // The getShiftAmountOperand() may create a new operand node or
1289 // return the existing one. If new operand is created we need
1290 // to update the parent node.
1291 // Do not try to legalize SAO here! It will be automatically legalized
1292 // in the next round.
1293 if (SAO != Op1)
1294 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1295 }
1296 }
1297 break;
1298 case ISD::FSHL:
1299 case ISD::FSHR:
1300 case ISD::SRL_PARTS:
1301 case ISD::SRA_PARTS:
1302 case ISD::SHL_PARTS: {
1303 // Legalizing shifts/rotates requires adjusting the shift amount
1304 // to the appropriate width.
1305 SDValue Op0 = Node->getOperand(0);
1306 SDValue Op1 = Node->getOperand(1);
1307 SDValue Op2 = Node->getOperand(2);
1308 if (!Op2.getValueType().isVector()) {
1309 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1310 // The getShiftAmountOperand() may create a new operand node or
1311 // return the existing one. If new operand is created we need
1312 // to update the parent node.
1313 if (SAO != Op2)
1314 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1315 }
1316 break;
1317 }
1318 }
1319
1320 if (NewNode != Node) {
1321 ReplaceNode(Node, NewNode);
1322 Node = NewNode;
1323 }
1324 switch (Action) {
1325 case TargetLowering::Legal:
1326 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1327 return;
1328 case TargetLowering::Custom:
1329 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1330 // FIXME: The handling for custom lowering with multiple results is
1331 // a complete mess.
1332 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1333 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1334 return;
1335
1336 if (Node->getNumValues() == 1) {
1337 // Verify the new types match the original. Glue is waived because
1338 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1339 assert((Res.getValueType() == Node->getValueType(0) ||
1340 Node->getValueType(0) == MVT::Glue) &&
1341 "Type mismatch for custom legalized operation");
1342 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1343 // We can just directly replace this node with the lowered value.
1344 ReplaceNode(SDValue(Node, 0), Res);
1345 return;
1346 }
1347
1348 SmallVector<SDValue, 8> ResultVals;
1349 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1350 // Verify the new types match the original. Glue is waived because
1351 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1352 assert((Res->getValueType(i) == Node->getValueType(i) ||
1353 Node->getValueType(i) == MVT::Glue) &&
1354 "Type mismatch for custom legalized operation");
1355 ResultVals.push_back(Res.getValue(i));
1356 }
1357 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1358 ReplaceNode(Node, ResultVals.data());
1359 return;
1360 }
1361 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1362 [[fallthrough]];
1363 case TargetLowering::Expand:
1364 if (ExpandNode(Node))
1365 return;
1366 [[fallthrough]];
1367 case TargetLowering::LibCall:
1368 ConvertNodeToLibcall(Node);
1369 return;
1370 case TargetLowering::Promote:
1371 PromoteNode(Node);
1372 return;
1373 }
1374 }
1375
1376 switch (Node->getOpcode()) {
1377 default:
1378#ifndef NDEBUG
1379 dbgs() << "NODE: ";
1380 Node->dump( &DAG);
1381 dbgs() << "\n";
1382#endif
1383 llvm_unreachable("Do not know how to legalize this operator!");
1384
1385 case ISD::CALLSEQ_START:
1386 case ISD::CALLSEQ_END:
1387 break;
1388 case ISD::LOAD:
1389 return LegalizeLoadOps(Node);
1390 case ISD::STORE:
1391 return LegalizeStoreOps(Node);
1392 }
1393}
1394
1395SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1396 SDValue Vec = Op.getOperand(0);
1397 SDValue Idx = Op.getOperand(1);
1398 SDLoc dl(Op);
1399
1400 // Before we generate a new store to a temporary stack slot, see if there is
1401 // already one that we can use. There often is because when we scalarize
1402 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1403 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1404 // the vector. If all are expanded here, we don't want one store per vector
1405 // element.
1406
1407 // Caches for hasPredecessorHelper
1410 Visited.insert(Op.getNode());
1411 Worklist.push_back(Idx.getNode());
1412 SDValue StackPtr, Ch;
1413 for (SDNode *User : Vec.getNode()->uses()) {
1414 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1415 if (ST->isIndexed() || ST->isTruncatingStore() ||
1416 ST->getValue() != Vec)
1417 continue;
1418
1419 // Make sure that nothing else could have stored into the destination of
1420 // this store.
1421 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1422 continue;
1423
1424 // If the index is dependent on the store we will introduce a cycle when
1425 // creating the load (the load uses the index, and by replacing the chain
1426 // we will make the index dependent on the load). Also, the store might be
1427 // dependent on the extractelement and introduce a cycle when creating
1428 // the load.
1429 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1430 ST->hasPredecessor(Op.getNode()))
1431 continue;
1432
1433 StackPtr = ST->getBasePtr();
1434 Ch = SDValue(ST, 0);
1435 break;
1436 }
1437 }
1438
1439 EVT VecVT = Vec.getValueType();
1440
1441 if (!Ch.getNode()) {
1442 // Store the value to a temporary stack slot, then LOAD the returned part.
1443 StackPtr = DAG.CreateStackTemporary(VecVT);
1445 StackPtr, DAG.getMachineFunction(), VecVT.isScalableVector());
1446 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1447 }
1448
1449 SDValue NewLoad;
1450 Align ElementAlignment =
1451 std::min(cast<StoreSDNode>(Ch)->getAlign(),
1452 DAG.getDataLayout().getPrefTypeAlign(
1453 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1454
1455 if (Op.getValueType().isVector()) {
1456 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1457 Op.getValueType(), Idx);
1458 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1459 MachinePointerInfo(), ElementAlignment);
1460 } else {
1461 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1462 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1464 ElementAlignment);
1465 }
1466
1467 // Replace the chain going out of the store, by the one out of the load.
1468 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1469
1470 // We introduced a cycle though, so update the loads operands, making sure
1471 // to use the original store's chain as an incoming chain.
1472 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1473 NewLoad->op_end());
1474 NewLoadOperands[0] = Ch;
1475 NewLoad =
1476 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1477 return NewLoad;
1478}
1479
1480SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1481 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1482
1483 SDValue Vec = Op.getOperand(0);
1484 SDValue Part = Op.getOperand(1);
1485 SDValue Idx = Op.getOperand(2);
1486 SDLoc dl(Op);
1487
1488 // Store the value to a temporary stack slot, then LOAD the returned part.
1489 EVT VecVT = Vec.getValueType();
1490 EVT SubVecVT = Part.getValueType();
1491 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1492 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1493 MachinePointerInfo PtrInfo =
1494 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1495
1496 // First store the whole vector.
1497 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1498
1499 // Then store the inserted part.
1500 SDValue SubStackPtr =
1501 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1502
1503 // Store the subvector.
1504 Ch = DAG.getStore(
1505 Ch, dl, Part, SubStackPtr,
1506 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1507
1508 // Finally, load the updated vector.
1509 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1510}
1511
1512SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1513 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1514 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1515 "Unexpected opcode!");
1516
1517 // We can't handle this case efficiently. Allocate a sufficiently
1518 // aligned object on the stack, store each operand into it, then load
1519 // the result as a vector.
1520 // Create the stack frame object.
1521 EVT VT = Node->getValueType(0);
1522 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1523 : Node->getOperand(0).getValueType();
1524 SDLoc dl(Node);
1525 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1526 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1527 MachinePointerInfo PtrInfo =
1528 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1529
1530 // Emit a store of each element to the stack slot.
1532 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1533 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1534
1535 // If the destination vector element type of a BUILD_VECTOR is narrower than
1536 // the source element type, only store the bits necessary.
1537 bool Truncate = isa<BuildVectorSDNode>(Node) &&
1538 MemVT.bitsLT(Node->getOperand(0).getValueType());
1539
1540 // Store (in the right endianness) the elements to memory.
1541 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1542 // Ignore undef elements.
1543 if (Node->getOperand(i).isUndef()) continue;
1544
1545 unsigned Offset = TypeByteSize*i;
1546
1547 SDValue Idx =
1548 DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl);
1549
1550 if (Truncate)
1551 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1552 Node->getOperand(i), Idx,
1553 PtrInfo.getWithOffset(Offset), MemVT));
1554 else
1555 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1556 Idx, PtrInfo.getWithOffset(Offset)));
1557 }
1558
1559 SDValue StoreChain;
1560 if (!Stores.empty()) // Not all undef elements?
1561 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1562 else
1563 StoreChain = DAG.getEntryNode();
1564
1565 // Result is a load from the stack slot.
1566 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1567}
1568
1569/// Bitcast a floating-point value to an integer value. Only bitcast the part
1570/// containing the sign bit if the target has no integer value capable of
1571/// holding all bits of the floating-point value.
1572void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1573 const SDLoc &DL,
1574 SDValue Value) const {
1575 EVT FloatVT = Value.getValueType();
1576 unsigned NumBits = FloatVT.getScalarSizeInBits();
1577 State.FloatVT = FloatVT;
1578 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1579 // Convert to an integer of the same size.
1580 if (TLI.isTypeLegal(IVT)) {
1581 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1582 State.SignMask = APInt::getSignMask(NumBits);
1583 State.SignBit = NumBits - 1;
1584 return;
1585 }
1586
1587 auto &DataLayout = DAG.getDataLayout();
1588 // Store the float to memory, then load the sign part out as an integer.
1589 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1590 // First create a temporary that is aligned for both the load and store.
1591 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1592 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1593 // Then store the float to it.
1594 State.FloatPtr = StackPtr;
1595 MachineFunction &MF = DAG.getMachineFunction();
1596 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1597 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1598 State.FloatPointerInfo);
1599
1600 SDValue IntPtr;
1601 if (DataLayout.isBigEndian()) {
1602 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1603 // Load out a legal integer with the same sign bit as the float.
1604 IntPtr = StackPtr;
1605 State.IntPointerInfo = State.FloatPointerInfo;
1606 } else {
1607 // Advance the pointer so that the loaded byte will contain the sign bit.
1608 unsigned ByteOffset = (NumBits / 8) - 1;
1609 IntPtr =
1610 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
1611 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1612 ByteOffset);
1613 }
1614
1615 State.IntPtr = IntPtr;
1616 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1617 State.IntPointerInfo, MVT::i8);
1618 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1619 State.SignBit = 7;
1620}
1621
1622/// Replace the integer value produced by getSignAsIntValue() with a new value
1623/// and cast the result back to a floating-point type.
1624SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1625 const SDLoc &DL,
1626 SDValue NewIntValue) const {
1627 if (!State.Chain)
1628 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1629
1630 // Override the part containing the sign bit in the value stored on the stack.
1631 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1632 State.IntPointerInfo, MVT::i8);
1633 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1634 State.FloatPointerInfo);
1635}
1636
1637SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1638 SDLoc DL(Node);
1639 SDValue Mag = Node->getOperand(0);
1640 SDValue Sign = Node->getOperand(1);
1641
1642 // Get sign bit into an integer value.
1643 FloatSignAsInt SignAsInt;
1644 getSignAsIntValue(SignAsInt, DL, Sign);
1645
1646 EVT IntVT = SignAsInt.IntValue.getValueType();
1647 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1648 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1649 SignMask);
1650
1651 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1652 EVT FloatVT = Mag.getValueType();
1653 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1654 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1655 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1656 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1657 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1658 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1659 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1660 }
1661
1662 // Transform Mag value to integer, and clear the sign bit.
1663 FloatSignAsInt MagAsInt;
1664 getSignAsIntValue(MagAsInt, DL, Mag);
1665 EVT MagVT = MagAsInt.IntValue.getValueType();
1666 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1667 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1668 ClearSignMask);
1669
1670 // Get the signbit at the right position for MagAsInt.
1671 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1672 EVT ShiftVT = IntVT;
1673 if (SignBit.getScalarValueSizeInBits() <
1674 ClearedSign.getScalarValueSizeInBits()) {
1675 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1676 ShiftVT = MagVT;
1677 }
1678 if (ShiftAmount > 0) {
1679 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1680 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1681 } else if (ShiftAmount < 0) {
1682 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1683 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1684 }
1685 if (SignBit.getScalarValueSizeInBits() >
1686 ClearedSign.getScalarValueSizeInBits()) {
1687 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1688 }
1689
1690 // Store the part with the modified sign and convert back to float.
1691 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1692 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1693}
1694
1695SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1696 // Get the sign bit as an integer.
1697 SDLoc DL(Node);
1698 FloatSignAsInt SignAsInt;
1699 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1700 EVT IntVT = SignAsInt.IntValue.getValueType();
1701
1702 // Flip the sign.
1703 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1704 SDValue SignFlip =
1705 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1706
1707 // Convert back to float.
1708 return modifySignAsInt(SignAsInt, DL, SignFlip);
1709}
1710
1711SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1712 SDLoc DL(Node);
1713 SDValue Value = Node->getOperand(0);
1714
1715 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1716 EVT FloatVT = Value.getValueType();
1717 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1718 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1719 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1720 }
1721
1722 // Transform value to integer, clear the sign bit and transform back.
1723 FloatSignAsInt ValueAsInt;
1724 getSignAsIntValue(ValueAsInt, DL, Value);
1725 EVT IntVT = ValueAsInt.IntValue.getValueType();
1726 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1727 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1728 ClearSignMask);
1729 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1730}
1731
1732void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1734 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1735 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1736 " not tell us which reg is the stack pointer!");
1737 SDLoc dl(Node);
1738 EVT VT = Node->getValueType(0);
1739 SDValue Tmp1 = SDValue(Node, 0);
1740 SDValue Tmp2 = SDValue(Node, 1);
1741 SDValue Tmp3 = Node->getOperand(2);
1742 SDValue Chain = Tmp1.getOperand(0);
1743
1744 // Chain the dynamic stack allocation so that it doesn't modify the stack
1745 // pointer when other instructions are using the stack.
1746 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1747
1748 SDValue Size = Tmp2.getOperand(1);
1749 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1750 Chain = SP.getValue(1);
1751 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1752 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1753 unsigned Opc =
1756
1757 Align StackAlign = TFL->getStackAlign();
1758 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1759 if (Alignment > StackAlign)
1760 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1761 DAG.getConstant(-Alignment.value(), dl, VT));
1762 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1763
1764 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
1765
1766 Results.push_back(Tmp1);
1767 Results.push_back(Tmp2);
1768}
1769
1770/// Emit a store/load combination to the stack. This stores
1771/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1772/// a load from the stack slot to DestVT, extending it if needed.
1773/// The resultant code need not be legal.
1774SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1775 EVT DestVT, const SDLoc &dl) {
1776 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1777}
1778
1779SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1780 EVT DestVT, const SDLoc &dl,
1781 SDValue Chain) {
1782 EVT SrcVT = SrcOp.getValueType();
1783 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1784 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1785
1786 // Don't convert with stack if the load/store is expensive.
1787 if ((SrcVT.bitsGT(SlotVT) &&
1788 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1789 (SlotVT.bitsLT(DestVT) &&
1790 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1791 return SDValue();
1792
1793 // Create the stack frame object.
1794 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1795 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1796 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1797
1798 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1799 int SPFI = StackPtrFI->getIndex();
1800 MachinePointerInfo PtrInfo =
1801 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1802
1803 // Emit a store to the stack slot. Use a truncstore if the input value is
1804 // later than DestVT.
1805 SDValue Store;
1806
1807 if (SrcVT.bitsGT(SlotVT))
1808 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1809 SlotVT, SrcAlign);
1810 else {
1811 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1812 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1813 }
1814
1815 // Result is a load from the stack slot.
1816 if (SlotVT.bitsEq(DestVT))
1817 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1818
1819 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1820 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1821 DestAlign);
1822}
1823
1824SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1825 SDLoc dl(Node);
1826 // Create a vector sized/aligned stack slot, store the value to element #0,
1827 // then load the whole vector back out.
1828 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1829
1830 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1831 int SPFI = StackPtrFI->getIndex();
1832
1833 SDValue Ch = DAG.getTruncStore(
1834 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1835 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1836 Node->getValueType(0).getVectorElementType());
1837 return DAG.getLoad(
1838 Node->getValueType(0), dl, Ch, StackPtr,
1839 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1840}
1841
1842static bool
1844 const TargetLowering &TLI, SDValue &Res) {
1845 unsigned NumElems = Node->getNumOperands();
1846 SDLoc dl(Node);
1847 EVT VT = Node->getValueType(0);
1848
1849 // Try to group the scalars into pairs, shuffle the pairs together, then
1850 // shuffle the pairs of pairs together, etc. until the vector has
1851 // been built. This will work only if all of the necessary shuffle masks
1852 // are legal.
1853
1854 // We do this in two phases; first to check the legality of the shuffles,
1855 // and next, assuming that all shuffles are legal, to create the new nodes.
1856 for (int Phase = 0; Phase < 2; ++Phase) {
1858 NewIntermedVals;
1859 for (unsigned i = 0; i < NumElems; ++i) {
1860 SDValue V = Node->getOperand(i);
1861 if (V.isUndef())
1862 continue;
1863
1864 SDValue Vec;
1865 if (Phase)
1866 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1867 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1868 }
1869
1870 while (IntermedVals.size() > 2) {
1871 NewIntermedVals.clear();
1872 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1873 // This vector and the next vector are shuffled together (simply to
1874 // append the one to the other).
1875 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1876
1877 SmallVector<int, 16> FinalIndices;
1878 FinalIndices.reserve(IntermedVals[i].second.size() +
1879 IntermedVals[i+1].second.size());
1880
1881 int k = 0;
1882 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1883 ++j, ++k) {
1884 ShuffleVec[k] = j;
1885 FinalIndices.push_back(IntermedVals[i].second[j]);
1886 }
1887 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1888 ++j, ++k) {
1889 ShuffleVec[k] = NumElems + j;
1890 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1891 }
1892
1893 SDValue Shuffle;
1894 if (Phase)
1895 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1896 IntermedVals[i+1].first,
1897 ShuffleVec);
1898 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1899 return false;
1900 NewIntermedVals.push_back(
1901 std::make_pair(Shuffle, std::move(FinalIndices)));
1902 }
1903
1904 // If we had an odd number of defined values, then append the last
1905 // element to the array of new vectors.
1906 if ((IntermedVals.size() & 1) != 0)
1907 NewIntermedVals.push_back(IntermedVals.back());
1908
1909 IntermedVals.swap(NewIntermedVals);
1910 }
1911
1912 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1913 "Invalid number of intermediate vectors");
1914 SDValue Vec1 = IntermedVals[0].first;
1915 SDValue Vec2;
1916 if (IntermedVals.size() > 1)
1917 Vec2 = IntermedVals[1].first;
1918 else if (Phase)
1919 Vec2 = DAG.getUNDEF(VT);
1920
1921 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1922 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1923 ShuffleVec[IntermedVals[0].second[i]] = i;
1924 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1925 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1926
1927 if (Phase)
1928 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1929 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1930 return false;
1931 }
1932
1933 return true;
1934}
1935
1936/// Expand a BUILD_VECTOR node on targets that don't
1937/// support the operation, but do support the resultant vector type.
1938SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1939 unsigned NumElems = Node->getNumOperands();
1940 SDValue Value1, Value2;
1941 SDLoc dl(Node);
1942 EVT VT = Node->getValueType(0);
1943 EVT OpVT = Node->getOperand(0).getValueType();
1944 EVT EltVT = VT.getVectorElementType();
1945
1946 // If the only non-undef value is the low element, turn this into a
1947 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1948 bool isOnlyLowElement = true;
1949 bool MoreThanTwoValues = false;
1950 bool isConstant = true;
1951 for (unsigned i = 0; i < NumElems; ++i) {
1952 SDValue V = Node->getOperand(i);
1953 if (V.isUndef())
1954 continue;
1955 if (i > 0)
1956 isOnlyLowElement = false;
1957 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1958 isConstant = false;
1959
1960 if (!Value1.getNode()) {
1961 Value1 = V;
1962 } else if (!Value2.getNode()) {
1963 if (V != Value1)
1964 Value2 = V;
1965 } else if (V != Value1 && V != Value2) {
1966 MoreThanTwoValues = true;
1967 }
1968 }
1969
1970 if (!Value1.getNode())
1971 return DAG.getUNDEF(VT);
1972
1973 if (isOnlyLowElement)
1974 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1975
1976 // If all elements are constants, create a load from the constant pool.
1977 if (isConstant) {
1979 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1980 if (ConstantFPSDNode *V =
1981 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1982 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1983 } else if (ConstantSDNode *V =
1984 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1985 if (OpVT==EltVT)
1986 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1987 else {
1988 // If OpVT and EltVT don't match, EltVT is not legal and the
1989 // element values have been promoted/truncated earlier. Undo this;
1990 // we don't want a v16i8 to become a v16i32 for example.
1991 const ConstantInt *CI = V->getConstantIntValue();
1992 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1993 CI->getZExtValue()));
1994 }
1995 } else {
1996 assert(Node->getOperand(i).isUndef());
1997 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1998 CV.push_back(UndefValue::get(OpNTy));
1999 }
2000 }
2002 SDValue CPIdx =
2003 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
2004 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2005 return DAG.getLoad(
2006 VT, dl, DAG.getEntryNode(), CPIdx,
2007 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2008 Alignment);
2009 }
2010
2011 SmallSet<SDValue, 16> DefinedValues;
2012 for (unsigned i = 0; i < NumElems; ++i) {
2013 if (Node->getOperand(i).isUndef())
2014 continue;
2015 DefinedValues.insert(Node->getOperand(i));
2016 }
2017
2018 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
2019 if (!MoreThanTwoValues) {
2020 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2021 for (unsigned i = 0; i < NumElems; ++i) {
2022 SDValue V = Node->getOperand(i);
2023 if (V.isUndef())
2024 continue;
2025 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2026 }
2027 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2028 // Get the splatted value into the low element of a vector register.
2029 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2030 SDValue Vec2;
2031 if (Value2.getNode())
2032 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2033 else
2034 Vec2 = DAG.getUNDEF(VT);
2035
2036 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2037 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2038 }
2039 } else {
2040 SDValue Res;
2041 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2042 return Res;
2043 }
2044 }
2045
2046 // Otherwise, we can't handle this case efficiently.
2047 return ExpandVectorBuildThroughStack(Node);
2048}
2049
2050SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2051 SDLoc DL(Node);
2052 EVT VT = Node->getValueType(0);
2053 SDValue SplatVal = Node->getOperand(0);
2054
2055 return DAG.getSplatBuildVector(VT, DL, SplatVal);
2056}
2057
2058// Expand a node into a call to a libcall, returning the value as the first
2059// result and the chain as the second. If the result value does not fit into a
2060// register, return the lo part and set the hi part to the by-reg argument in
2061// the first. If it does fit into a single register, return the result and
2062// leave the Hi part unset.
2063std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2065 bool isSigned) {
2066 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2067 TLI.getPointerTy(DAG.getDataLayout()));
2068
2069 EVT RetVT = Node->getValueType(0);
2070 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2071
2072 // By default, the input chain to this libcall is the entry node of the
2073 // function. If the libcall is going to be emitted as a tail call then
2074 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2075 // node which is being folded has a non-entry input chain.
2076 SDValue InChain = DAG.getEntryNode();
2077
2078 // isTailCall may be true since the callee does not reference caller stack
2079 // frame. Check if it's in the right position and that the return types match.
2080 SDValue TCChain = InChain;
2081 const Function &F = DAG.getMachineFunction().getFunction();
2082 bool isTailCall =
2083 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2084 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2085 if (isTailCall)
2086 InChain = TCChain;
2087
2089 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2090 CLI.setDebugLoc(SDLoc(Node))
2091 .setChain(InChain)
2092 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2093 std::move(Args))
2094 .setTailCall(isTailCall)
2095 .setSExtResult(signExtend)
2096 .setZExtResult(!signExtend)
2097 .setIsPostTypeLegalization(true);
2098
2099 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2100
2101 if (!CallInfo.second.getNode()) {
2102 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2103 // It's a tailcall, return the chain (which is the DAG root).
2104 return {DAG.getRoot(), DAG.getRoot()};
2105 }
2106
2107 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2108 return CallInfo;
2109}
2110
2111std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2112 bool isSigned) {
2115 for (const SDValue &Op : Node->op_values()) {
2116 EVT ArgVT = Op.getValueType();
2117 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2118 Entry.Node = Op;
2119 Entry.Ty = ArgTy;
2120 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2121 Entry.IsZExt = !Entry.IsSExt;
2122 Args.push_back(Entry);
2123 }
2124
2125 return ExpandLibCall(LC, Node, std::move(Args), isSigned);
2126}
2127
2128void SelectionDAGLegalize::ExpandFrexpLibCall(
2130 SDLoc dl(Node);
2131 EVT VT = Node->getValueType(0);
2132 EVT ExpVT = Node->getValueType(1);
2133
2134 SDValue FPOp = Node->getOperand(0);
2135
2136 EVT ArgVT = FPOp.getValueType();
2137 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2138
2140 FPArgEntry.Node = FPOp;
2141 FPArgEntry.Ty = ArgTy;
2142
2143 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2144 TargetLowering::ArgListEntry PtrArgEntry;
2145 PtrArgEntry.Node = StackSlot;
2146 PtrArgEntry.Ty = PointerType::get(*DAG.getContext(),
2147 DAG.getDataLayout().getAllocaAddrSpace());
2148
2149 TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2150
2152 auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false);
2153
2154 // FIXME: Get type of int for libcall declaration and cast
2155
2156 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2157 auto PtrInfo =
2158 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
2159
2160 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2161 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2162 LoadExp.getValue(1), DAG.getRoot());
2163 DAG.setRoot(OutputChain);
2164
2165 Results.push_back(Call);
2166 Results.push_back(LoadExp);
2167}
2168
2169void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2170 RTLIB::Libcall LC,
2172 if (LC == RTLIB::UNKNOWN_LIBCALL)
2173 llvm_unreachable("Can't create an unknown libcall!");
2174
2175 if (Node->isStrictFPOpcode()) {
2176 EVT RetVT = Node->getValueType(0);
2179 // FIXME: This doesn't support tail calls.
2180 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2181 Ops, CallOptions,
2182 SDLoc(Node),
2183 Node->getOperand(0));
2184 Results.push_back(Tmp.first);
2185 Results.push_back(Tmp.second);
2186 } else {
2187 SDValue Tmp = ExpandLibCall(LC, Node, false).first;
2188 Results.push_back(Tmp);
2189 }
2190}
2191
2192/// Expand the node to a libcall based on the result type.
2193void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2194 RTLIB::Libcall Call_F32,
2195 RTLIB::Libcall Call_F64,
2196 RTLIB::Libcall Call_F80,
2197 RTLIB::Libcall Call_F128,
2198 RTLIB::Libcall Call_PPCF128,
2200 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2201 Call_F32, Call_F64, Call_F80,
2202 Call_F128, Call_PPCF128);
2203 ExpandFPLibCall(Node, LC, Results);
2204}
2205
2206SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2207 RTLIB::Libcall Call_I8,
2208 RTLIB::Libcall Call_I16,
2209 RTLIB::Libcall Call_I32,
2210 RTLIB::Libcall Call_I64,
2211 RTLIB::Libcall Call_I128) {
2212 RTLIB::Libcall LC;
2213 switch (Node->getSimpleValueType(0).SimpleTy) {
2214 default: llvm_unreachable("Unexpected request for libcall!");
2215 case MVT::i8: LC = Call_I8; break;
2216 case MVT::i16: LC = Call_I16; break;
2217 case MVT::i32: LC = Call_I32; break;
2218 case MVT::i64: LC = Call_I64; break;
2219 case MVT::i128: LC = Call_I128; break;
2220 }
2221 return ExpandLibCall(LC, Node, isSigned).first;
2222}
2223
2224/// Expand the node to a libcall based on first argument type (for instance
2225/// lround and its variant).
2226void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2227 RTLIB::Libcall Call_F32,
2228 RTLIB::Libcall Call_F64,
2229 RTLIB::Libcall Call_F80,
2230 RTLIB::Libcall Call_F128,
2231 RTLIB::Libcall Call_PPCF128,
2233 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2235 Call_F32, Call_F64, Call_F80,
2236 Call_F128, Call_PPCF128);
2237 ExpandFPLibCall(Node, LC, Results);
2238}
2239
2240/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2241void
2242SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2244 unsigned Opcode = Node->getOpcode();
2245 bool isSigned = Opcode == ISD::SDIVREM;
2246
2247 RTLIB::Libcall LC;
2248 switch (Node->getSimpleValueType(0).SimpleTy) {
2249 default: llvm_unreachable("Unexpected request for libcall!");
2250 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2251 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2252 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2253 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2254 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2255 }
2256
2257 // The input chain to this libcall is the entry node of the function.
2258 // Legalizing the call will automatically add the previous call to the
2259 // dependence.
2260 SDValue InChain = DAG.getEntryNode();
2261
2262 EVT RetVT = Node->getValueType(0);
2263 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2264
2267 for (const SDValue &Op : Node->op_values()) {
2268 EVT ArgVT = Op.getValueType();
2269 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2270 Entry.Node = Op;
2271 Entry.Ty = ArgTy;
2272 Entry.IsSExt = isSigned;
2273 Entry.IsZExt = !isSigned;
2274 Args.push_back(Entry);
2275 }
2276
2277 // Also pass the return address of the remainder.
2278 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2279 Entry.Node = FIPtr;
2280 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2281 Entry.IsSExt = isSigned;
2282 Entry.IsZExt = !isSigned;
2283 Args.push_back(Entry);
2284
2285 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2286 TLI.getPointerTy(DAG.getDataLayout()));
2287
2288 SDLoc dl(Node);
2290 CLI.setDebugLoc(dl)
2291 .setChain(InChain)
2292 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2293 std::move(Args))
2294 .setSExtResult(isSigned)
2295 .setZExtResult(!isSigned);
2296
2297 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2298
2299 // Remainder is loaded back from the stack frame.
2300 SDValue Rem =
2301 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2302 Results.push_back(CallInfo.first);
2303 Results.push_back(Rem);
2304}
2305
2306/// Return true if sincos libcall is available.
2308 RTLIB::Libcall LC;
2309 switch (Node->getSimpleValueType(0).SimpleTy) {
2310 default: llvm_unreachable("Unexpected request for libcall!");
2311 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2312 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2313 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2314 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2315 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2316 }
2317 return TLI.getLibcallName(LC) != nullptr;
2318}
2319
2320/// Only issue sincos libcall if both sin and cos are needed.
2321static bool useSinCos(SDNode *Node) {
2322 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2323 ? ISD::FCOS : ISD::FSIN;
2324
2325 SDValue Op0 = Node->getOperand(0);
2326 for (const SDNode *User : Op0.getNode()->uses()) {
2327 if (User == Node)
2328 continue;
2329 // The other user might have been turned into sincos already.
2330 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2331 return true;
2332 }
2333 return false;
2334}
2335
2336/// Issue libcalls to sincos to compute sin / cos pairs.
2337void
2338SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2340 RTLIB::Libcall LC;
2341 switch (Node->getSimpleValueType(0).SimpleTy) {
2342 default: llvm_unreachable("Unexpected request for libcall!");
2343 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2344 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2345 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2346 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2347 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2348 }
2349
2350 // The input chain to this libcall is the entry node of the function.
2351 // Legalizing the call will automatically add the previous call to the
2352 // dependence.
2353 SDValue InChain = DAG.getEntryNode();
2354
2355 EVT RetVT = Node->getValueType(0);
2356 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2357
2360
2361 // Pass the argument.
2362 Entry.Node = Node->getOperand(0);
2363 Entry.Ty = RetTy;
2364 Entry.IsSExt = false;
2365 Entry.IsZExt = false;
2366 Args.push_back(Entry);
2367
2368 // Pass the return address of sin.
2369 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2370 Entry.Node = SinPtr;
2371 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2372 Entry.IsSExt = false;
2373 Entry.IsZExt = false;
2374 Args.push_back(Entry);
2375
2376 // Also pass the return address of the cos.
2377 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2378 Entry.Node = CosPtr;
2379 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2380 Entry.IsSExt = false;
2381 Entry.IsZExt = false;
2382 Args.push_back(Entry);
2383
2384 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2385 TLI.getPointerTy(DAG.getDataLayout()));
2386
2387 SDLoc dl(Node);
2389 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2390 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2391 std::move(Args));
2392
2393 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2394
2395 Results.push_back(
2396 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2397 Results.push_back(
2398 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2399}
2400
2401SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2402 SDLoc dl(Node);
2403 EVT VT = Node->getValueType(0);
2404 SDValue X = Node->getOperand(0);
2405 SDValue N = Node->getOperand(1);
2406 EVT ExpVT = N.getValueType();
2407 EVT AsIntVT = VT.changeTypeToInteger();
2408 if (AsIntVT == EVT()) // TODO: How to handle f80?
2409 return SDValue();
2410
2411 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2412 return SDValue();
2413
2414 SDNodeFlags NSW;
2415 NSW.setNoSignedWrap(true);
2416 SDNodeFlags NUW_NSW;
2417 NUW_NSW.setNoUnsignedWrap(true);
2418 NUW_NSW.setNoSignedWrap(true);
2419
2420 EVT SetCCVT =
2421 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2423
2424 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2425 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2426 const int Precision = APFloat::semanticsPrecision(FltSem);
2427
2428 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2429 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2430
2431 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2432
2433 const APFloat One(FltSem, "1.0");
2434 APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2435
2436 // Offset by precision to avoid denormal range.
2437 APFloat ScaleDownK =
2438 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2439
2440 // TODO: Should really introduce control flow and use a block for the >
2441 // MaxExp, < MinExp cases
2442
2443 // First, handle exponents Exp > MaxExp and scale down.
2444 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT);
2445
2446 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW);
2447 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2448 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal);
2449 SDValue DecN1 =
2450 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);
2451
2452 SDValue ScaleUpTwice =
2453 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT);
2454
2455 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2456 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal);
2457 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal);
2458
2459 SDValue SelectN_Big =
2460 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0);
2461 SDValue SelectX_Big =
2462 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0);
2463
2464 // Now handle exponents Exp < MinExp
2465 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT);
2466
2467 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2468 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2469
2470 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW);
2471
2472 SDValue ClampMinVal =
2473 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2474 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal);
2475 SDValue IncN1 =
2476 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);
2477
2478 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2479 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal);
2480 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal);
2481
2482 SDValue ScaleDownTwice = DAG.getSetCC(
2483 dl, SetCCVT, N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2484 ISD::SETULT);
2485
2486 SDValue SelectN_Small =
2487 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0);
2488 SDValue SelectX_Small =
2489 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);
2490
2491 // Now combine the two out of range exponent handling cases with the base
2492 // case.
2493 SDValue NewX = DAG.getNode(
2494 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,
2495 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X));
2496
2497 SDValue NewN = DAG.getNode(
2498 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,
2499 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N));
2500
2501 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW);
2502
2503 SDValue ExponentShiftAmt =
2504 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2505 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2506
2507 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy,
2508 ExponentShiftAmt, NUW_NSW);
2509 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);
2510 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP);
2511}
2512
2513SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2514 SDLoc dl(Node);
2515 SDValue Val = Node->getOperand(0);
2516 EVT VT = Val.getValueType();
2517 EVT ExpVT = Node->getValueType(1);
2518 EVT AsIntVT = VT.changeTypeToInteger();
2519 if (AsIntVT == EVT()) // TODO: How to handle f80?
2520 return SDValue();
2521
2523 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2524 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2525 const unsigned BitSize = VT.getScalarSizeInBits();
2526
2527 // TODO: Could introduce control flow and skip over the denormal handling.
2528
2529 // scale_up = fmul value, scalbn(1.0, precision + 1)
2530 // extracted_exp = (bitcast value to uint) >> precision - 1
2531 // biased_exp = extracted_exp + min_exp
2532 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2533 //
2534 // is_denormal = val < smallest_normalized
2535 // computed_fract = is_denormal ? scale_up : extracted_fract
2536 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2537 //
2538 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2539 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2540
2541 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2542 APFloat::getSmallestNormalized(FltSem, true).bitcastToAPInt(), dl,
2543 AsIntVT);
2544
2545 SDValue SmallestNormalizedInt = DAG.getConstant(
2546 APFloat::getSmallestNormalized(FltSem, false).bitcastToAPInt(), dl,
2547 AsIntVT);
2548
2549 // Masks out the exponent bits.
2550 SDValue ExpMask =
2551 DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2552
2553 // Mask out the exponent part of the value.
2554 //
2555 // e.g, for f32 FractSignMaskVal = 0x807fffff
2556 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);
2557 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2558
2559 APInt SignMaskVal = APInt::getSignedMaxValue(BitSize);
2560 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2561
2562 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2563
2564 const APFloat One(FltSem, "1.0");
2565 // Scale a possible denormal input.
2566 // e.g., for f64, 0x1p+54
2567 APFloat ScaleUpKVal =
2568 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2569
2570 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2571 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK);
2572
2573 EVT SetCCVT =
2574 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2575
2576 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);
2577
2578 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask);
2579
2580 SDValue AddNegSmallestNormal =
2581 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);
2582 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2583 NegSmallestNormalizedInt, ISD::SETULE);
2584
2585 SDValue IsDenormal =
2586 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
2587
2588 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2589 SDValue Zero = DAG.getConstant(0, dl, ExpVT);
2590
2591 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
2592 SDValue ScaledSelect =
2593 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);
2594
2595 SDValue ExpMaskScaled =
2596 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);
2597
2598 SDValue ScaledValue =
2599 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);
2600
2601 // Extract the exponent bits.
2602 SDValue ExponentShiftAmt =
2603 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2604 SDValue ShiftedExp =
2605 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);
2606 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2607
2608 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp);
2609 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2610 SDValue DenormalExpBias =
2611 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);
2612
2613 SDValue MaskedFractAsInt =
2614 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);
2615 const APFloat Half(FltSem, "0.5");
2616 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2617 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf);
2618 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);
2619
2620 SDValue ComputedExp =
2621 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);
2622
2623 SDValue Result0 =
2624 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract);
2625
2626 SDValue Result1 =
2627 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);
2628
2629 return DAG.getMergeValues({Result0, Result1}, dl);
2630}
2631
2632/// This function is responsible for legalizing a
2633/// INT_TO_FP operation of the specified operand when the target requests that
2634/// we expand it. At this point, we know that the result and operand types are
2635/// legal for the target.
2636SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2637 SDValue &Chain) {
2638 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2639 Node->getOpcode() == ISD::SINT_TO_FP);
2640 EVT DestVT = Node->getValueType(0);
2641 SDLoc dl(Node);
2642 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2643 SDValue Op0 = Node->getOperand(OpNo);
2644 EVT SrcVT = Op0.getValueType();
2645
2646 // TODO: Should any fast-math-flags be set for the created nodes?
2647 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2648 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2649 (DestVT.bitsLE(MVT::f64) ||
2650 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2652 DestVT))) {
2653 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2654 "expansion\n");
2655
2656 // Get the stack frame index of a 8 byte buffer.
2657 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2658
2659 SDValue Lo = Op0;
2660 // if signed map to unsigned space
2661 if (isSigned) {
2662 // Invert sign bit (signed to unsigned mapping).
2663 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2664 DAG.getConstant(0x80000000u, dl, MVT::i32));
2665 }
2666 // Initial hi portion of constructed double.
2667 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2668
2669 // If this a big endian target, swap the lo and high data.
2670 if (DAG.getDataLayout().isBigEndian())
2671 std::swap(Lo, Hi);
2672
2673 SDValue MemChain = DAG.getEntryNode();
2674
2675 // Store the lo of the constructed double.
2676 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2678 // Store the hi of the constructed double.
2679 SDValue HiPtr =
2680 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
2681 SDValue Store2 =
2682 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2683 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2684
2685 // load the constructed double
2686 SDValue Load =
2687 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2688 // FP constant to bias correct the final result
2689 SDValue Bias = DAG.getConstantFP(
2690 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2691 : llvm::bit_cast<double>(0x4330000000000000ULL),
2692 dl, MVT::f64);
2693 // Subtract the bias and get the final result.
2694 SDValue Sub;
2696 if (Node->isStrictFPOpcode()) {
2697 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2698 {Node->getOperand(0), Load, Bias});
2699 Chain = Sub.getValue(1);
2700 if (DestVT != Sub.getValueType()) {
2701 std::pair<SDValue, SDValue> ResultPair;
2702 ResultPair =
2703 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2704 Result = ResultPair.first;
2705 Chain = ResultPair.second;
2706 }
2707 else
2708 Result = Sub;
2709 } else {
2710 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2711 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2712 }
2713 return Result;
2714 }
2715
2716 if (isSigned)
2717 return SDValue();
2718
2719 // TODO: Generalize this for use with other types.
2720 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2721 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2722 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2723 // For unsigned conversions, convert them to signed conversions using the
2724 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2725 // should be valid for i32->f32 as well.
2726
2727 // More generally this transform should be valid if there are 3 more bits
2728 // in the integer type than the significand. Rounding uses the first bit
2729 // after the width of the significand and the OR of all bits after that. So
2730 // we need to be able to OR the shifted out bit into one of the bits that
2731 // participate in the OR.
2732
2733 // TODO: This really should be implemented using a branch rather than a
2734 // select. We happen to get lucky and machinesink does the right
2735 // thing most of the time. This would be a good candidate for a
2736 // pseudo-op, or, even better, for whole-function isel.
2737 EVT SetCCVT = getSetCCResultType(SrcVT);
2738
2739 SDValue SignBitTest = DAG.getSetCC(
2740 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2741
2742 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2743 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2744 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2745 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2746 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2747 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2748
2749 SDValue Slow, Fast;
2750 if (Node->isStrictFPOpcode()) {
2751 // In strict mode, we must avoid spurious exceptions, and therefore
2752 // must make sure to only emit a single STRICT_SINT_TO_FP.
2753 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2754 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2755 { Node->getOperand(0), InCvt });
2756 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2757 { Fast.getValue(1), Fast, Fast });
2758 Chain = Slow.getValue(1);
2759 // The STRICT_SINT_TO_FP inherits the exception mode from the
2760 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2761 // never raise any exception.
2763 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2764 Fast->setFlags(Flags);
2765 Flags.setNoFPExcept(true);
2766 Slow->setFlags(Flags);
2767 } else {
2768 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2769 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2770 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2771 }
2772
2773 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2774 }
2775
2776 // Don't expand it if there isn't cheap fadd.
2777 if (!TLI.isOperationLegalOrCustom(
2778 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2779 return SDValue();
2780
2781 // The following optimization is valid only if every value in SrcVT (when
2782 // treated as signed) is representable in DestVT. Check that the mantissa
2783 // size of DestVT is >= than the number of bits in SrcVT -1.
2784 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2785 SrcVT.getSizeInBits() - 1 &&
2786 "Cannot perform lossless SINT_TO_FP!");
2787
2788 SDValue Tmp1;
2789 if (Node->isStrictFPOpcode()) {
2790 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2791 { Node->getOperand(0), Op0 });
2792 } else
2793 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2794
2795 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2796 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2797 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2798 Four = DAG.getIntPtrConstant(4, dl);
2799 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2800 SignSet, Four, Zero);
2801
2802 // If the sign bit of the integer is set, the large number will be treated
2803 // as a negative number. To counteract this, the dynamic code adds an
2804 // offset depending on the data type.
2805 uint64_t FF;
2806 switch (SrcVT.getSimpleVT().SimpleTy) {
2807 default:
2808 return SDValue();
2809 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2810 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2811 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2812 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2813 }
2814 if (DAG.getDataLayout().isLittleEndian())
2815 FF <<= 32;
2816 Constant *FudgeFactor = ConstantInt::get(
2817 Type::getInt64Ty(*DAG.getContext()), FF);
2818
2819 SDValue CPIdx =
2820 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2821 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2822 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2823 Alignment = commonAlignment(Alignment, 4);
2824 SDValue FudgeInReg;
2825 if (DestVT == MVT::f32)
2826 FudgeInReg = DAG.getLoad(
2827 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2828 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2829 Alignment);
2830 else {
2831 SDValue Load = DAG.getExtLoad(
2832 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2833 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2834 Alignment);
2835 HandleSDNode Handle(Load);
2836 LegalizeOp(Load.getNode());
2837 FudgeInReg = Handle.getValue();
2838 }
2839
2840 if (Node->isStrictFPOpcode()) {
2841 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2842 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2843 Chain = Result.getValue(1);
2844 return Result;
2845 }
2846
2847 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2848}
2849
2850/// This function is responsible for legalizing a
2851/// *INT_TO_FP operation of the specified operand when the target requests that
2852/// we promote it. At this point, we know that the result and operand types are
2853/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2854/// operation that takes a larger input.
2855void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2857 bool IsStrict = N->isStrictFPOpcode();
2858 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2859 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2860 EVT DestVT = N->getValueType(0);
2861 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2862 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2863 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2864
2865 // First step, figure out the appropriate *INT_TO_FP operation to use.
2866 EVT NewInTy = LegalOp.getValueType();
2867
2868 unsigned OpToUse = 0;
2869
2870 // Scan for the appropriate larger type to use.
2871 while (true) {
2872 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2873 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2874
2875 // If the target supports SINT_TO_FP of this type, use it.
2876 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2877 OpToUse = SIntOp;
2878 break;
2879 }
2880 if (IsSigned)
2881 continue;
2882
2883 // If the target supports UINT_TO_FP of this type, use it.
2884 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2885 OpToUse = UIntOp;
2886 break;
2887 }
2888
2889 // Otherwise, try a larger type.
2890 }
2891
2892 // Okay, we found the operation and type to use. Zero extend our input to the
2893 // desired type then run the operation on it.
2894 if (IsStrict) {
2895 SDValue Res =
2896 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2897 {N->getOperand(0),
2898 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2899 dl, NewInTy, LegalOp)});
2900 Results.push_back(Res);
2901 Results.push_back(Res.getValue(1));
2902 return;
2903 }
2904
2905 Results.push_back(
2906 DAG.getNode(OpToUse, dl, DestVT,
2907 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2908 dl, NewInTy, LegalOp)));
2909}
2910
2911/// This function is responsible for legalizing a
2912/// FP_TO_*INT operation of the specified operand when the target requests that
2913/// we promote it. At this point, we know that the result and operand types are
2914/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2915/// operation that returns a larger result.
2916void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2918 bool IsStrict = N->isStrictFPOpcode();
2919 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2920 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2921 EVT DestVT = N->getValueType(0);
2922 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2923 // First step, figure out the appropriate FP_TO*INT operation to use.
2924 EVT NewOutTy = DestVT;
2925
2926 unsigned OpToUse = 0;
2927
2928 // Scan for the appropriate larger type to use.
2929 while (true) {
2930 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2931 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2932
2933 // A larger signed type can hold all unsigned values of the requested type,
2934 // so using FP_TO_SINT is valid
2935 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2936 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2937 break;
2938
2939 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2940 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2941 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2942 break;
2943
2944 // Otherwise, try a larger type.
2945 }
2946
2947 // Okay, we found the operation and type to use.
2949 if (IsStrict) {
2950 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2951 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2952 } else
2953 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2954
2955 // Truncate the result of the extended FP_TO_*INT operation to the desired
2956 // size.
2957 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2958 Results.push_back(Trunc);
2959 if (IsStrict)
2960 Results.push_back(Operation.getValue(1));
2961}
2962
2963/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2964/// the result and operand types are legal and there must be a legal
2965/// FP_TO_*INT_SAT operation for a larger result type.
2966SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2967 const SDLoc &dl) {
2968 unsigned Opcode = Node->getOpcode();
2969
2970 // Scan for the appropriate larger type to use.
2971 EVT NewOutTy = Node->getValueType(0);
2972 while (true) {
2973 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2974 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2975
2976 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2977 break;
2978 }
2979
2980 // Saturation width is determined by second operand, so we don't have to
2981 // perform any fixup and can directly truncate the result.
2982 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2983 Node->getOperand(1));
2984 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2985}
2986
2987/// Open code the operations for PARITY of the specified operation.
2988SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2989 EVT VT = Op.getValueType();
2990 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2991 unsigned Sz = VT.getScalarSizeInBits();
2992
2993 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2995 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2996 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2997 } else {
2998 Result = Op;
2999 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
3000 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
3001 DAG.getConstant(1ULL << (--i), dl, ShVT));
3002 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
3003 }
3004 }
3005
3006 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
3007}
3008
3009bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3010 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3012 SDLoc dl(Node);
3013 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3014 bool NeedInvert;
3015 switch (Node->getOpcode()) {
3016 case ISD::ABS:
3017 if ((Tmp1 = TLI.expandABS(Node, DAG)))
3018 Results.push_back(Tmp1);
3019 break;
3020 case ISD::ABDS:
3021 case ISD::ABDU:
3022 if ((Tmp1 = TLI.expandABD(Node, DAG)))
3023 Results.push_back(Tmp1);
3024 break;
3025 case ISD::CTPOP:
3026 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
3027 Results.push_back(Tmp1);
3028 break;
3029 case ISD::CTLZ:
3031 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
3032 Results.push_back(Tmp1);
3033 break;
3034 case ISD::CTTZ:
3036 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
3037 Results.push_back(Tmp1);
3038 break;
3039 case ISD::BITREVERSE:
3040 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
3041 Results.push_back(Tmp1);
3042 break;
3043 case ISD::BSWAP:
3044 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
3045 Results.push_back(Tmp1);
3046 break;
3047 case ISD::PARITY:
3048 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
3049 break;
3050 case ISD::FRAMEADDR:
3051 case ISD::RETURNADDR:
3053 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3054 break;
3055 case ISD::EH_DWARF_CFA: {
3056 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
3057 TLI.getPointerTy(DAG.getDataLayout()));
3058 SDValue Offset = DAG.getNode(ISD::ADD, dl,
3059 CfaArg.getValueType(),
3060 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
3061 CfaArg.getValueType()),
3062 CfaArg);
3063 SDValue FA = DAG.getNode(
3064 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
3065 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3066 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
3067 FA, Offset));
3068 break;
3069 }
3070 case ISD::GET_ROUNDING:
3071 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
3072 Results.push_back(Node->getOperand(0));
3073 break;
3074 case ISD::EH_RETURN:
3075 case ISD::EH_LABEL:
3076 case ISD::PREFETCH:
3077 case ISD::VAEND:
3079 // If the target didn't expand these, there's nothing to do, so just
3080 // preserve the chain and be done.
3081 Results.push_back(Node->getOperand(0));
3082 break;
3085 // If the target didn't expand this, just return 'zero' and preserve the
3086 // chain.
3087 Results.append(Node->getNumValues() - 1,
3088 DAG.getConstant(0, dl, Node->getValueType(0)));
3089 Results.push_back(Node->getOperand(0));
3090 break;
3092 // If the target didn't expand this, just return 'zero' and preserve the
3093 // chain.
3094 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3095 Results.push_back(Node->getOperand(0));
3096 break;
3097 case ISD::ATOMIC_LOAD: {
3098 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3099 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
3100 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3101 SDValue Swap = DAG.getAtomicCmpSwap(
3102 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3103 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3104 cast<AtomicSDNode>(Node)->getMemOperand());
3105 Results.push_back(Swap.getValue(0));
3106 Results.push_back(Swap.getValue(1));
3107 break;
3108 }
3109 case ISD::ATOMIC_STORE: {
3110 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3111 SDValue Swap = DAG.getAtomic(
3112 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
3113 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3114 cast<AtomicSDNode>(Node)->getMemOperand());
3115 Results.push_back(Swap.getValue(1));
3116 break;
3117 }
3119 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3120 // splits out the success value as a comparison. Expanding the resulting
3121 // ATOMIC_CMP_SWAP will produce a libcall.
3122 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3123 SDValue Res = DAG.getAtomicCmpSwap(
3124 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3125 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3126 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
3127
3128 SDValue ExtRes = Res;
3129 SDValue LHS = Res;
3130 SDValue RHS = Node->getOperand(1);
3131
3132 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
3133 EVT OuterType = Node->getValueType(0);
3134 switch (TLI.getExtendForAtomicOps()) {
3135 case ISD::SIGN_EXTEND:
3136 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
3137 DAG.getValueType(AtomicType));
3138 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
3139 Node->getOperand(2), DAG.getValueType(AtomicType));
3140 ExtRes = LHS;
3141 break;
3142 case ISD::ZERO_EXTEND:
3143 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
3144 DAG.getValueType(AtomicType));
3145 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3146 ExtRes = LHS;
3147 break;
3148 case ISD::ANY_EXTEND:
3149 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3150 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3151 break;
3152 default:
3153 llvm_unreachable("Invalid atomic op extension");
3154 }
3155
3157 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
3158
3159 Results.push_back(ExtRes.getValue(0));
3160 Results.push_back(Success);
3161 Results.push_back(Res.getValue(1));
3162 break;
3163 }
3164 case ISD::ATOMIC_LOAD_SUB: {
3165 SDLoc DL(Node);
3166 EVT VT = Node->getValueType(0);
3167 SDValue RHS = Node->getOperand(2);
3168 AtomicSDNode *AN = cast<AtomicSDNode>(Node);
3169 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3170 cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT())
3171 RHS = RHS->getOperand(0);
3172 SDValue NewRHS =
3173 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
3174 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(),
3175 Node->getOperand(0), Node->getOperand(1),
3176 NewRHS, AN->getMemOperand());
3177 Results.push_back(Res);
3178 Results.push_back(Res.getValue(1));
3179 break;
3180 }
3182 ExpandDYNAMIC_STACKALLOC(Node, Results);
3183 break;
3184 case ISD::MERGE_VALUES:
3185 for (unsigned i = 0; i < Node->getNumValues(); i++)
3186 Results.push_back(Node->getOperand(i));
3187 break;
3188 case ISD::UNDEF: {
3189 EVT VT = Node->getValueType(0);
3190 if (VT.isInteger())
3191 Results.push_back(DAG.getConstant(0, dl, VT));
3192 else {
3193 assert(VT.isFloatingPoint() && "Unknown value type!");
3194 Results.push_back(DAG.getConstantFP(0, dl, VT));
3195 }
3196 break;
3197 }
3199 // When strict mode is enforced we can't do expansion because it
3200 // does not honor the "strict" properties. Only libcall is allowed.
3201 if (TLI.isStrictFPEnabled())
3202 break;
3203 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3204 // since this operation is more efficient than stack operation.
3205 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3206 Node->getValueType(0))
3207 == TargetLowering::Legal)
3208 break;
3209 // We fall back to use stack operation when the FP_ROUND operation
3210 // isn't available.
3211 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
3212 Node->getValueType(0), dl,
3213 Node->getOperand(0)))) {
3214 ReplaceNode(Node, Tmp1.getNode());
3215 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3216 return true;
3217 }
3218 break;
3219 case ISD::FP_ROUND: {
3220 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3221 Results.push_back(Tmp1);
3222 break;
3223 }
3224
3226 }
3227 case ISD::BITCAST:
3228 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3229 Node->getValueType(0), dl)))
3230 Results.push_back(Tmp1);
3231 break;
3233 // When strict mode is enforced we can't do expansion because it
3234 // does not honor the "strict" properties. Only libcall is allowed.
3235 if (TLI.isStrictFPEnabled())
3236 break;
3237 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3238 // since this operation is more efficient than stack operation.
3239 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3240 Node->getValueType(0))
3241 == TargetLowering::Legal)
3242 break;
3243 // We fall back to use stack operation when the FP_EXTEND operation
3244 // isn't available.
3245 if ((Tmp1 = EmitStackConvert(
3246 Node->getOperand(1), Node->getOperand(1).getValueType(),
3247 Node->getValueType(0), dl, Node->getOperand(0)))) {
3248 ReplaceNode(Node, Tmp1.getNode());
3249 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3250 return true;
3251 }
3252 break;
3253 case ISD::FP_EXTEND: {
3254 SDValue Op = Node->getOperand(0);
3255 EVT SrcVT = Op.getValueType();
3256 EVT DstVT = Node->getValueType(0);
3257 if (SrcVT.getScalarType() == MVT::bf16) {
3258 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op));
3259 break;
3260 }
3261
3262 if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl)))
3263 Results.push_back(Tmp1);
3264 break;
3265 }
3266 case ISD::BF16_TO_FP: {
3267 // Always expand bf16 to f32 casts, they lower to ext + shift.
3268 //
3269 // Note that the operand of this code can be bf16 or an integer type in case
3270 // bf16 is not supported on the target and was softened.
3271 SDValue Op = Node->getOperand(0);
3272 if (Op.getValueType() == MVT::bf16) {
3273 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3274 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3275 } else {
3276 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3277 }
3278 Op = DAG.getNode(
3279 ISD::SHL, dl, MVT::i32, Op,
3280 DAG.getConstant(16, dl,
3281 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3282 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3283 // Add fp_extend in case the output is bigger than f32.
3284 if (Node->getValueType(0) != MVT::f32)
3285 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
3286 Results.push_back(Op);
3287 break;
3288 }
3289 case ISD::FP_TO_BF16: {
3290 SDValue Op = Node->getOperand(0);
3291 if (Op.getValueType() != MVT::f32)
3292 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3293 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3294 // Certain SNaNs will turn into infinities if we do a simple shift right.
3295 if (!DAG.isKnownNeverSNaN(Op)) {
3296 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags());
3297 }
3298 Op = DAG.getNode(
3299 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3300 DAG.getConstant(16, dl,
3301 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3302 // The result of this node can be bf16 or an integer type in case bf16 is
3303 // not supported on the target and was softened to i16 for storage.
3304 if (Node->getValueType(0) == MVT::bf16) {
3305 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3306 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3307 } else {
3308 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0));
3309 }
3310 Results.push_back(Op);
3311 break;
3312 }
3314 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3315 EVT VT = Node->getValueType(0);
3316
3317 // An in-register sign-extend of a boolean is a negation:
3318 // 'true' (1) sign-extended is -1.
3319 // 'false' (0) sign-extended is 0.
3320 // However, we must mask the high bits of the source operand because the
3321 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3322
3323 // TODO: Do this for vectors too?
3324 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3325 SDValue One = DAG.getConstant(1, dl, VT);
3326 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3327 SDValue Zero = DAG.getConstant(0, dl, VT);
3328 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3329 Results.push_back(Neg);
3330 break;
3331 }
3332
3333 // NOTE: we could fall back on load/store here too for targets without
3334 // SRA. However, it is doubtful that any exist.
3335 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3336 unsigned BitsDiff = VT.getScalarSizeInBits() -
3337 ExtraVT.getScalarSizeInBits();
3338 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3339 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3340 Node->getOperand(0), ShiftCst);
3341 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3342 Results.push_back(Tmp1);
3343 break;
3344 }
3345 case ISD::UINT_TO_FP:
3347 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
3348 Results.push_back(Tmp1);
3349 if (Node->isStrictFPOpcode())
3350 Results.push_back(Tmp2);
3351 break;
3352 }
3353 [[fallthrough]];
3354 case ISD::SINT_TO_FP:
3356 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3357 Results.push_back(Tmp1);
3358 if (Node->isStrictFPOpcode())
3359 Results.push_back(Tmp2);
3360 }
3361 break;
3362 case ISD::FP_TO_SINT:
3363 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3364 Results.push_back(Tmp1);
3365 break;
3367 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
3368 ReplaceNode(Node, Tmp1.getNode());
3369 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3370 return true;
3371 }
3372 break;
3373 case ISD::FP_TO_UINT:
3374 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
3375 Results.push_back(Tmp1);
3376 break;
3378 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
3379 // Relink the chain.
3380 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
3381 // Replace the new UINT result.
3382 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
3383 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3384 return true;
3385 }
3386 break;
3389 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
3390 break;
3391 case ISD::VAARG:
3392 Results.push_back(DAG.expandVAArg(Node));
3393 Results.push_back(Results[0].getValue(1));
3394 break;
3395 case ISD::VACOPY:
3396 Results.push_back(DAG.expandVACopy(Node));
3397 break;
3399 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3400 // This must be an access of the only element. Return it.
3401 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3402 Node->getOperand(0));
3403 else
3404 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3405 Results.push_back(Tmp1);
3406 break;
3408 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3409 break;
3411 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3412 break;
3414 Results.push_back(ExpandVectorBuildThroughStack(Node));
3415 break;
3417 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3418 break;
3420 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3421 Node->getOperand(1),
3422 Node->getOperand(2), dl));
3423 break;
3424 case ISD::VECTOR_SHUFFLE: {
3425 SmallVector<int, 32> NewMask;
3426 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3427
3428 EVT VT = Node->getValueType(0);
3429 EVT EltVT = VT.getVectorElementType();
3430 SDValue Op0 = Node->getOperand(0);
3431 SDValue Op1 = Node->getOperand(1);
3432 if (!TLI.isTypeLegal(EltVT)) {
3433 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3434
3435 // BUILD_VECTOR operands are allowed to be wider than the element type.
3436 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3437 // it.
3438 if (NewEltVT.bitsLT(EltVT)) {
3439 // Convert shuffle node.
3440 // If original node was v4i64 and the new EltVT is i32,
3441 // cast operands to v8i32 and re-build the mask.
3442
3443 // Calculate new VT, the size of the new VT should be equal to original.
3444 EVT NewVT =
3445 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3446 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3447 assert(NewVT.bitsEq(VT));
3448
3449 // cast operands to new VT
3450 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3451 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3452
3453 // Convert the shuffle mask
3454 unsigned int factor =
3456
3457 // EltVT gets smaller
3458 assert(factor > 0);
3459
3460 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3461 if (Mask[i] < 0) {
3462 for (unsigned fi = 0; fi < factor; ++fi)
3463 NewMask.push_back(Mask[i]);
3464 }
3465 else {
3466 for (unsigned fi = 0; fi < factor; ++fi)
3467 NewMask.push_back(Mask[i]*factor+fi);
3468 }
3469 }
3470 Mask = NewMask;
3471 VT = NewVT;
3472 }
3473 EltVT = NewEltVT;
3474 }
3475 unsigned NumElems = VT.getVectorNumElements();
3477 for (unsigned i = 0; i != NumElems; ++i) {
3478 if (Mask[i] < 0) {
3479 Ops.push_back(DAG.getUNDEF(EltVT));
3480 continue;
3481 }
3482 unsigned Idx = Mask[i];
3483 if (Idx < NumElems)
3484 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3485 DAG.getVectorIdxConstant(Idx, dl)));
3486 else
3487 Ops.push_back(
3488 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3489 DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3490 }
3491
3492 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3493 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3494 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3495 Results.push_back(Tmp1);
3496 break;
3497 }
3498 case ISD::VECTOR_SPLICE: {
3499 Results.push_back(TLI.expandVectorSplice(Node, DAG));
3500 break;
3501 }
3502 case ISD::EXTRACT_ELEMENT: {
3503 EVT OpTy = Node->getOperand(0).getValueType();
3504 if (Node->getConstantOperandVal(1)) {
3505 // 1 -> Hi
3506 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3507 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3508 TLI.getShiftAmountTy(
3509 Node->getOperand(0).getValueType(),
3510 DAG.getDataLayout())));
3511 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3512 } else {
3513 // 0 -> Lo
3514 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3515 Node->getOperand(0));
3516 }
3517 Results.push_back(Tmp1);
3518 break;
3519 }
3520 case ISD::STACKSAVE:
3521 // Expand to CopyFromReg if the target set
3522 // StackPointerRegisterToSaveRestore.
3523 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3524 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3525 Node->getValueType(0)));
3526 Results.push_back(Results[0].getValue(1));
3527 } else {
3528 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3529 Results.push_back(Node->getOperand(0));
3530 }
3531 break;
3532 case ISD::STACKRESTORE:
3533 // Expand to CopyToReg if the target set
3534 // StackPointerRegisterToSaveRestore.
3535 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3536 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3537 Node->getOperand(1)));
3538 } else {
3539 Results.push_back(Node->getOperand(0));
3540 }
3541 break;
3543 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3544 Results.push_back(Results[0].getValue(0));
3545 break;
3546 case ISD::FCOPYSIGN:
3547 Results.push_back(ExpandFCOPYSIGN(Node));
3548 break;
3549 case ISD::FNEG:
3550 Results.push_back(ExpandFNEG(Node));
3551 break;
3552 case ISD::FABS:
3553 Results.push_back(ExpandFABS(Node));
3554 break;
3555 case ISD::IS_FPCLASS: {
3556 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1));
3557 if (SDValue Expanded =
3558 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3559 Test, Node->getFlags(), SDLoc(Node), DAG))
3560 Results.push_back(Expanded);
3561 break;
3562 }
3563 case ISD::SMIN:
3564 case ISD::SMAX:
3565 case ISD::UMIN:
3566 case ISD::UMAX: {
3567 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3568 ISD::CondCode Pred;
3569 switch (Node->getOpcode()) {
3570 default: llvm_unreachable("How did we get here?");
3571 case ISD::SMAX: Pred = ISD::SETGT; break;
3572 case ISD::SMIN: Pred = ISD::SETLT; break;
3573 case ISD::UMAX: Pred = ISD::SETUGT; break;
3574 case ISD::UMIN: Pred = ISD::SETULT; break;
3575 }
3576 Tmp1 = Node->getOperand(0);
3577 Tmp2 = Node->getOperand(1);
3578 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3579 Results.push_back(Tmp1);
3580 break;
3581 }
3582 case ISD::FMINNUM:
3583 case ISD::FMAXNUM: {
3584 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3585 Results.push_back(Expanded);
3586 break;
3587 }
3588 case ISD::FSIN:
3589 case ISD::FCOS: {
3590 EVT VT = Node->getValueType(0);
3591 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3592 // fcos which share the same operand and both are used.
3593 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3595 && useSinCos(Node)) {
3596 SDVTList VTs = DAG.getVTList(VT, VT);
3597 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3598 if (Node->getOpcode() == ISD::FCOS)
3599 Tmp1 = Tmp1.getValue(1);
3600 Results.push_back(Tmp1);
3601 }
3602 break;
3603 }
3604 case ISD::FLDEXP:
3605 case ISD::STRICT_FLDEXP: {
3606 EVT VT = Node->getValueType(0);
3608 // Use the LibCall instead, it is very likely faster
3609 // FIXME: Use separate LibCall action.
3610 if (TLI.getLibcallName(LC))
3611 break;
3612
3613 if (SDValue Expanded = expandLdexp(Node)) {
3614 Results.push_back(Expanded);
3615 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3616 Results.push_back(Expanded.getValue(1));
3617 }
3618
3619 break;
3620 }
3621 case ISD::FFREXP: {
3622 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));
3623 // Use the LibCall instead, it is very likely faster
3624 // FIXME: Use separate LibCall action.
3625 if (TLI.getLibcallName(LC))
3626 break;
3627
3628 if (SDValue Expanded = expandFrexp(Node)) {
3629 Results.push_back(Expanded);
3630 Results.push_back(Expanded.getValue(1));
3631 }
3632 break;
3633 }
3634 case ISD::FMAD:
3635 llvm_unreachable("Illegal fmad should never be formed");
3636
3637 case ISD::FP16_TO_FP:
3638 if (Node->getValueType(0) != MVT::f32) {
3639 // We can extend to types bigger than f32 in two steps without changing
3640 // the result. Since "f16 -> f32" is much more commonly available, give
3641 // CodeGen the option of emitting that before resorting to a libcall.
3642 SDValue Res =
3643 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3644 Results.push_back(
3645 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3646 }
3647 break;
3649 if (Node->getValueType(0) != MVT::f32) {
3650 // We can extend to types bigger than f32 in two steps without changing
3651 // the result. Since "f16 -> f32" is much more commonly available, give
3652 // CodeGen the option of emitting that before resorting to a libcall.
3653 SDValue Res =
3654 DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other},
3655 {Node->getOperand(0), Node->getOperand(1)});
3656 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3657 {Node->getValueType(0), MVT::Other},
3658 {Res.getValue(1), Res});
3659 Results.push_back(Res);
3660 Results.push_back(Res.getValue(1));
3661 }
3662 break;
3663 case ISD::FP_TO_FP16:
3664 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3665 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3666 SDValue Op = Node->getOperand(0);
3667 MVT SVT = Op.getSimpleValueType();
3668 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3669 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3670 // Under fastmath, we can expand this node into a fround followed by
3671 // a float-half conversion.
3672 SDValue FloatVal =
3673 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3674 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3675 Results.push_back(
3676 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3677 }
3678 }
3679 break;
3680 case ISD::ConstantFP: {
3681 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3682 // Check to see if this FP immediate is already legal.
3683 // If this is a legal constant, turn it into a TargetConstantFP node.
3684 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3685 DAG.shouldOptForSize()))
3686 Results.push_back(ExpandConstantFP(CFP, true));
3687 break;
3688 }
3689 case ISD::Constant: {
3690 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3691 Results.push_back(ExpandConstant(CP));
3692 break;
3693 }
3694 case ISD::FSUB: {
3695 EVT VT = Node->getValueType(0);
3696 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3697 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3698 const SDNodeFlags Flags = Node->getFlags();
3699 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3700 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3701 Results.push_back(Tmp1);
3702 }
3703 break;
3704 }
3705 case ISD::SUB: {
3706 EVT VT = Node->getValueType(0);
3707 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3708 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3709 "Don't know how to expand this subtraction!");
3710 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3711 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3712 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3713 break;
3714 }
3715 case ISD::UREM:
3716 case ISD::SREM:
3717 if (TLI.expandREM(Node, Tmp1, DAG))
3718 Results.push_back(Tmp1);
3719 break;
3720 case ISD::UDIV:
3721 case ISD::SDIV: {
3722 bool isSigned = Node->getOpcode() == ISD::SDIV;
3723 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3724 EVT VT = Node->getValueType(0);
3725 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3726 SDVTList VTs = DAG.getVTList(VT, VT);
3727 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3728 Node->getOperand(1));
3729 Results.push_back(Tmp1);
3730 }
3731 break;
3732 }
3733 case ISD::MULHU:
3734 case ISD::MULHS: {
3735 unsigned ExpandOpcode =
3736 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3737 EVT VT = Node->getValueType(0);
3738 SDVTList VTs = DAG.getVTList(VT, VT);
3739
3740 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3741 Node->getOperand(1));
3742 Results.push_back(Tmp1.getValue(1));
3743 break;
3744 }
3745 case ISD::UMUL_LOHI:
3746 case ISD::SMUL_LOHI: {
3747 SDValue LHS = Node->getOperand(0);
3748 SDValue RHS = Node->getOperand(1);
3749 MVT VT = LHS.getSimpleValueType();
3750 unsigned MULHOpcode =
3751 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3752
3753 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3754 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3755 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3756 break;
3757 }
3758
3760 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3761 assert(TLI.isTypeLegal(HalfType));
3762 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3763 HalfType, DAG,
3764 TargetLowering::MulExpansionKind::Always)) {
3765 for (unsigned i = 0; i < 2; ++i) {
3766 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3767 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3768 SDValue Shift = DAG.getConstant(
3769 HalfType.getScalarSizeInBits(), dl,
3770 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3771 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3772 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3773 }
3774 break;
3775 }
3776 break;
3777 }
3778 case ISD::MUL: {
3779 EVT VT = Node->getValueType(0);
3780 SDVTList VTs = DAG.getVTList(VT, VT);
3781 // See if multiply or divide can be lowered using two-result operations.
3782 // We just need the low half of the multiply; try both the signed
3783 // and unsigned forms. If the target supports both SMUL_LOHI and
3784 // UMUL_LOHI, form a preference by checking which forms of plain
3785 // MULH it supports.
3786 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3787 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3788 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3789 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3790 unsigned OpToUse = 0;
3791 if (HasSMUL_LOHI && !HasMULHS) {
3792 OpToUse = ISD::SMUL_LOHI;
3793 } else if (HasUMUL_LOHI && !HasMULHU) {
3794 OpToUse = ISD::UMUL_LOHI;
3795 } else if (HasSMUL_LOHI) {
3796 OpToUse = ISD::SMUL_LOHI;
3797 } else if (HasUMUL_LOHI) {
3798 OpToUse = ISD::UMUL_LOHI;
3799 }
3800 if (OpToUse) {
3801 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3802 Node->getOperand(1)));
3803 break;
3804 }
3805
3806 SDValue Lo, Hi;
3807 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3808 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3809 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3810 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3811 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3812 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3813 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3814 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3815 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3816 SDValue Shift =
3817 DAG.getConstant(HalfType.getSizeInBits(), dl,
3818 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3819 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3820 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3821 }
3822 break;
3823 }
3824 case ISD::FSHL:
3825 case ISD::FSHR:
3826 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3827 Results.push_back(Expanded);
3828 break;
3829 case ISD::ROTL:
3830 case ISD::ROTR:
3831 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3832 Results.push_back(Expanded);
3833 break;
3834 case ISD::SADDSAT:
3835 case ISD::UADDSAT:
3836 case ISD::SSUBSAT:
3837 case ISD::USUBSAT:
3838 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3839 break;
3840 case ISD::SSHLSAT:
3841 case ISD::USHLSAT:
3842 Results.push_back(TLI.expandShlSat(Node, DAG));
3843 break;
3844 case ISD::SMULFIX:
3845 case ISD::SMULFIXSAT:
3846 case ISD::UMULFIX:
3847 case ISD::UMULFIXSAT:
3848 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3849 break;
3850 case ISD::SDIVFIX:
3851 case ISD::SDIVFIXSAT:
3852 case ISD::UDIVFIX:
3853 case ISD::UDIVFIXSAT:
3854 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3855 Node->getOperand(0),
3856 Node->getOperand(1),
3857 Node->getConstantOperandVal(2),
3858 DAG)) {
3859 Results.push_back(V);
3860 break;
3861 }
3862 // FIXME: We might want to retry here with a wider type if we fail, if that
3863 // type is legal.
3864 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3865 // <= 128 (which is the case for all of the default Embedded-C types),
3866 // we will only get here with types and scales that we could always expand
3867 // if we were allowed to generate libcalls to division functions of illegal
3868 // type. But we cannot do that.
3869 llvm_unreachable("Cannot expand DIVFIX!");
3870 case ISD::UADDO_CARRY:
3871 case ISD::USUBO_CARRY: {
3872 SDValue LHS = Node->getOperand(0);
3873 SDValue RHS = Node->getOperand(1);
3874 SDValue Carry = Node->getOperand(2);
3875
3876 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3877
3878 // Initial add of the 2 operands.
3879 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3880 EVT VT = LHS.getValueType();
3881 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3882
3883 // Initial check for overflow.
3884 EVT CarryType = Node->getValueType(1);
3885 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3887 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3888
3889 // Add of the sum and the carry.
3890 SDValue One = DAG.getConstant(1, dl, VT);
3891 SDValue CarryExt =
3892 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3893 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3894
3895 // Second check for overflow. If we are adding, we can only overflow if the
3896 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3897 // If we are subtracting, we can only overflow if the initial sum is 0 and
3898 // the carry is set, resulting in a new sum of all 1s.
3899 SDValue Zero = DAG.getConstant(0, dl, VT);
3900 SDValue Overflow2 =
3901 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3902 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3903 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3904 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3905
3906 SDValue ResultCarry =
3907 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3908
3909 Results.push_back(Sum2);
3910 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3911 break;
3912 }
3913 case ISD::SADDO:
3914 case ISD::SSUBO: {
3915 SDValue Result, Overflow;
3916 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3917 Results.push_back(Result);
3918 Results.push_back(Overflow);
3919 break;
3920 }
3921 case ISD::UADDO:
3922 case ISD::USUBO: {
3923 SDValue Result, Overflow;
3924 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3925 Results.push_back(Result);
3926 Results.push_back(Overflow);
3927 break;
3928 }
3929 case ISD::UMULO:
3930 case ISD::SMULO: {
3931 SDValue Result, Overflow;
3932 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3933 Results.push_back(Result);
3934 Results.push_back(Overflow);
3935 }
3936 break;
3937 }
3938 case ISD::BUILD_PAIR: {
3939 EVT PairTy = Node->getValueType(0);
3940 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3941 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3942 Tmp2 = DAG.getNode(
3943 ISD::SHL, dl, PairTy, Tmp2,
3944 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3945 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3946 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3947 break;
3948 }
3949 case ISD::SELECT:
3950 Tmp1 = Node->getOperand(0);
3951 Tmp2 = Node->getOperand(1);
3952 Tmp3 = Node->getOperand(2);
3953 if (Tmp1.getOpcode() == ISD::SETCC) {
3954 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3955 Tmp2, Tmp3,
3956 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3957 } else {
3958 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3959 DAG.getConstant(0, dl, Tmp1.getValueType()),
3960 Tmp2, Tmp3, ISD::SETNE);
3961 }
3962 Tmp1->setFlags(Node->getFlags());
3963 Results.push_back(Tmp1);
3964 break;
3965 case ISD::BR_JT: {
3966 SDValue Chain = Node->getOperand(0);
3967 SDValue Table = Node->getOperand(1);
3968 SDValue Index = Node->getOperand(2);
3969 int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex();
3970
3971 const DataLayout &TD = DAG.getDataLayout();
3972 EVT PTy = TLI.getPointerTy(TD);
3973
3974 unsigned EntrySize =
3975 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3976
3977 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3978 // This transformation needs to be done here since otherwise the MIPS
3979 // backend will end up emitting a three instruction multiply sequence
3980 // instead of a single shift and MSP430 will call a runtime function.
3981 if (llvm::isPowerOf2_32(EntrySize))
3982 Index = DAG.getNode(
3983 ISD::SHL, dl, Index.getValueType(), Index,
3984 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3985 else
3986 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3987 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3988 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3989 Index, Table);
3990
3991 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3992 SDValue LD = DAG.getExtLoad(
3993 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3994 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3995 Addr = LD;
3996 if (TLI.isJumpTableRelative()) {
3997 // For PIC, the sequence is:
3998 // BRIND(load(Jumptable + index) + RelocBase)
3999 // RelocBase can be JumpTable, GOT or some sort of global base.
4000 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
4001 TLI.getPICJumpTableRelocBase(Table, DAG));
4002 }
4003
4004 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);
4005 Results.push_back(Tmp1);
4006 break;
4007 }
4008 case ISD::BRCOND:
4009 // Expand brcond's setcc into its constituent parts and create a BR_CC
4010 // Node.
4011 Tmp1 = Node->getOperand(0);
4012 Tmp2 = Node->getOperand(1);
4013 if (Tmp2.getOpcode() == ISD::SETCC &&
4014 TLI.isOperationLegalOrCustom(ISD::BR_CC,
4015 Tmp2.getOperand(0).getValueType())) {
4016 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
4017 Tmp2.getOperand(0), Tmp2.getOperand(1),
4018 Node->getOperand(2));
4019 } else {
4020 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4021 if (Tmp2.isUndef() ||
4022 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1))))
4023 Tmp3 = Tmp2;
4024 else
4025 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
4026 DAG.getConstant(1, dl, Tmp2.getValueType()));
4027 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
4028 DAG.getCondCode(ISD::SETNE), Tmp3,
4029 DAG.getConstant(0, dl, Tmp3.getValueType()),
4030 Node->getOperand(2));
4031 }
4032 Results.push_back(Tmp1);
4033 break;
4034 case ISD::SETCC:
4035 case ISD::VP_SETCC:
4036 case ISD::STRICT_FSETCC:
4037 case ISD::STRICT_FSETCCS: {
4038 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4039 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4040 Node->getOpcode() == ISD::STRICT_FSETCCS;
4041 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4042 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4043 unsigned Offset = IsStrict ? 1 : 0;
4044 Tmp1 = Node->getOperand(0 + Offset);
4045 Tmp2 = Node->getOperand(1 + Offset);
4046 Tmp3 = Node->getOperand(2 + Offset);
4047 SDValue Mask, EVL;
4048 if (IsVP) {
4049 Mask = Node->getOperand(3 + Offset);
4050 EVL = Node->getOperand(4 + Offset);
4051 }
4052 bool Legalized = TLI.LegalizeSetCCCondCode(
4053 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4054 Chain, IsSignaling);
4055
4056 if (Legalized) {
4057 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4058 // condition code, create a new SETCC node.
4059 if (Tmp3.getNode()) {
4060 if (IsStrict) {
4061 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
4062 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
4063 Chain = Tmp1.getValue(1);
4064 } else if (IsVP) {
4065 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
4066 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
4067 } else {
4068 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
4069 Tmp2, Tmp3, Node->getFlags());
4070 }
4071 }
4072
4073 // If we expanded the SETCC by inverting the condition code, then wrap
4074 // the existing SETCC in a NOT to restore the intended condition.
4075 if (NeedInvert) {
4076 if (!IsVP)
4077 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
4078 else
4079 Tmp1 =
4080 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
4081 }
4082
4083 Results.push_back(Tmp1);
4084 if (IsStrict)
4085 Results.push_back(Chain);
4086
4087 break;
4088 }
4089
4090 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4091 // understand if this code is useful for strict nodes.
4092 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4093
4094 // Otherwise, SETCC for the given comparison type must be completely
4095 // illegal; expand it into a SELECT_CC.
4096 // FIXME: This drops the mask/evl for VP_SETCC.
4097 EVT VT = Node->getValueType(0);
4098 EVT Tmp1VT = Tmp1.getValueType();
4099 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
4100 DAG.getBoolConstant(true, dl, VT, Tmp1VT),
4101 DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
4102 Tmp1->setFlags(Node->getFlags());
4103 Results.push_back(Tmp1);
4104 break;
4105 }
4106 case ISD::SELECT_CC: {
4107 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4108 Tmp1 = Node->getOperand(0); // LHS
4109 Tmp2 = Node->getOperand(1); // RHS
4110 Tmp3 = Node->getOperand(2); // True
4111 Tmp4 = Node->getOperand(3); // False
4112 EVT VT = Node->getValueType(0);
4113 SDValue Chain;
4114 SDValue CC = Node->getOperand(4);
4115 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
4116
4117 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
4118 // If the condition code is legal, then we need to expand this
4119 // node using SETCC and SELECT.
4120 EVT CmpVT = Tmp1.getValueType();
4121 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4122 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4123 "expanded.");
4124 EVT CCVT = getSetCCResultType(CmpVT);
4125 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
4126 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
4127 break;
4128 }
4129
4130 // SELECT_CC is legal, so the condition code must not be.
4131 bool Legalized = false;
4132 // Try to legalize by inverting the condition. This is for targets that
4133 // might support an ordered version of a condition, but not the unordered
4134 // version (or vice versa).
4135 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
4136 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
4137 // Use the new condition code and swap true and false
4138 Legalized = true;
4139 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4140 Tmp1->setFlags(Node->getFlags());
4141 } else {
4142 // If The inverse is not legal, then try to swap the arguments using
4143 // the inverse condition code.
4145 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
4146 // The swapped inverse condition is legal, so swap true and false,
4147 // lhs and rhs.
4148 Legalized = true;
4149 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4150 Tmp1->setFlags(Node->getFlags());
4151 }
4152 }
4153
4154 if (!Legalized) {
4155 Legalized = TLI.LegalizeSetCCCondCode(
4156 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
4157 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4158
4159 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4160
4161 // If we expanded the SETCC by inverting the condition code, then swap
4162 // the True/False operands to match.
4163 if (NeedInvert)
4164 std::swap(Tmp3, Tmp4);
4165
4166 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4167 // condition code, create a new SELECT_CC node.
4168 if (CC.getNode()) {
4169 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
4170 Tmp1, Tmp2, Tmp3, Tmp4, CC);
4171 } else {
4172 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
4173 CC = DAG.getCondCode(ISD::SETNE);
4174 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
4175 Tmp2, Tmp3, Tmp4, CC);
4176 }
4177 Tmp1->setFlags(Node->getFlags());
4178 }
4179 Results.push_back(Tmp1);
4180 break;
4181 }
4182 case ISD::BR_CC: {
4183 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4184 SDValue Chain;
4185 Tmp1 = Node->getOperand(0); // Chain
4186 Tmp2 = Node->getOperand(2); // LHS
4187 Tmp3 = Node->getOperand(3); // RHS
4188 Tmp4 = Node->getOperand(1); // CC
4189
4190 bool Legalized = TLI.LegalizeSetCCCondCode(
4191 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
4192 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4193 (void)Legalized;
4194 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4195
4196 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4197 // node.
4198 if (Tmp4.getNode()) {
4199 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4200
4201 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4202 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4203 } else {
4204 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
4205 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
4206 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4207 Tmp2, Tmp3, Node->getOperand(4));
4208 }
4209 Results.push_back(Tmp1);
4210 break;
4211 }
4212 case ISD::BUILD_VECTOR:
4213 Results.push_back(ExpandBUILD_VECTOR(Node));
4214 break;
4215 case ISD::SPLAT_VECTOR:
4216 Results.push_back(ExpandSPLAT_VECTOR(Node));
4217 break;
4218 case ISD::SRA:
4219 case ISD::SRL:
4220 case ISD::SHL: {
4221 // Scalarize vector SRA/SRL/SHL.
4222 EVT VT = Node->getValueType(0);
4223 assert(VT.isVector() && "Unable to legalize non-vector shift");
4224 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4225 unsigned NumElem = VT.getVectorNumElements();
4226
4228 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4229 SDValue Ex =
4231 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
4232 SDValue Sh =
4234 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
4235 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
4236 VT.getScalarType(), Ex, Sh));
4237 }
4238
4239 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
4240 Results.push_back(Result);
4241 break;
4242 }
4245 case ISD::VECREDUCE_ADD:
4246 case ISD::VECREDUCE_MUL:
4247 case ISD::VECREDUCE_AND:
4248 case ISD::VECREDUCE_OR:
4249 case ISD::VECREDUCE_XOR:
4258 Results.push_back(TLI.expandVecReduce(Node, DAG));
4259 break;
4261 case ISD::GlobalAddress:
4264 case ISD::ConstantPool:
4265 case ISD::JumpTable:
4269 // FIXME: Custom lowering for these operations shouldn't return null!
4270 // Return true so that we don't call ConvertNodeToLibcall which also won't
4271 // do anything.
4272 return true;
4273 }
4274
4275 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4276 // FIXME: We were asked to expand a strict floating-point operation,
4277 // but there is currently no expansion implemented that would preserve
4278 // the "strict" properties. For now, we just fall back to the non-strict
4279 // version if that is legal on the target. The actual mutation of the
4280 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4281 switch (Node->getOpcode()) {
4282 default:
4283 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4284 Node->getValueType(0))
4285 == TargetLowering::Legal)
4286 return true;
4287 break;
4288 case ISD::STRICT_FSUB: {
4289 if (TLI.getStrictFPOperationAction(
4290 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
4291 return true;
4292 if (TLI.getStrictFPOperationAction(
4293 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
4294 break;
4295
4296 EVT VT = Node->getValueType(0);
4297 const SDNodeFlags Flags = Node->getFlags();
4298 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
4299 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
4300 {Node->getOperand(0), Node->getOperand(1), Neg},
4301 Flags);
4302
4303 Results.push_back(Fadd);
4304 Results.push_back(Fadd.getValue(1));
4305 break;
4306 }
4309 case ISD::STRICT_LRINT:
4310 case ISD::STRICT_LLRINT:
4311 case ISD::STRICT_LROUND:
4313 // These are registered by the operand type instead of the value
4314 // type. Reflect that here.
4315 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4316 Node->getOperand(1).getValueType())
4317 == TargetLowering::Legal)
4318 return true;
4319 break;
4320 }
4321 }
4322
4323 // Replace the original node with the legalized result.
4324 if (Results.empty()) {
4325 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4326 return false;
4327 }
4328
4329 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4330 ReplaceNode(Node, Results.data());
4331 return true;
4332}
4333
4334void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4335 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4337 SDLoc dl(Node);
4338 // FIXME: Check flags on the node to see if we can use a finite call.
4339 unsigned Opc = Node->getOpcode();
4340 switch (Opc) {
4341 case ISD::ATOMIC_FENCE: {
4342 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4343 // FIXME: handle "fence singlethread" more efficiently.
4345
4347 CLI.setDebugLoc(dl)
4348 .setChain(Node->getOperand(0))
4349 .setLibCallee(
4350 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4351 DAG.getExternalSymbol("__sync_synchronize",
4352 TLI.getPointerTy(DAG.getDataLayout())),
4353 std::move(Args));
4354
4355 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4356
4357 Results.push_back(CallResult.second);
4358 break;
4359 }
4360 // By default, atomic intrinsics are marked Legal and lowered. Targets
4361 // which don't support them directly, however, may want libcalls, in which
4362 // case they mark them Expand, and we get here.
4363 case ISD::ATOMIC_SWAP:
4375 case ISD::ATOMIC_CMP_SWAP: {
4376 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4377 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
4378 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4379 EVT RetVT = Node->getValueType(0);
4382 if (TLI.getLibcallName(LC)) {
4383 // If outline atomic available, prepare its arguments and expand.
4384 Ops.append(Node->op_begin() + 2, Node->op_end());
4385 Ops.push_back(Node->getOperand(1));
4386
4387 } else {
4388 LC = RTLIB::getSYNC(Opc, VT);
4389 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4390 "Unexpected atomic op or value type!");
4391 // Arguments for expansion to sync libcall
4392 Ops.append(Node->op_begin() + 1, Node->op_end());
4393 }
4394 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4395 Ops, CallOptions,
4396 SDLoc(Node),
4397 Node->getOperand(0));
4398 Results.push_back(Tmp.first);
4399 Results.push_back(Tmp.second);
4400 break;
4401 }
4402 case ISD::TRAP: {
4403 // If this operation is not supported, lower it to 'abort()' call
4406 CLI.setDebugLoc(dl)
4407 .setChain(Node->getOperand(0))
4408 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4409 DAG.getExternalSymbol(
4410 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4411 std::move(Args));
4412 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4413
4414 Results.push_back(CallResult.second);
4415 break;
4416 }
4417 case ISD::FMINNUM:
4419 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4420 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4421 RTLIB::FMIN_PPCF128, Results);
4422 break;
4423 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4424 // libcall legalization for these nodes, but there is no default expasion for
4425 // these nodes either (see PR63267 for example).
4426 case ISD::FMAXNUM:
4428 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4429 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4430 RTLIB::FMAX_PPCF128, Results);
4431 break;
4432 case ISD::FSQRT:
4433 case ISD::STRICT_FSQRT:
4434 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4435 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4436 RTLIB::SQRT_PPCF128, Results);
4437 break;
4438 case ISD::FCBRT:
4439 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4440 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4441 RTLIB::CBRT_PPCF128, Results);
4442 break;
4443 case ISD::FSIN:
4444 case ISD::STRICT_FSIN:
4445 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4446 RTLIB::SIN_F80, RTLIB::SIN_F128,
4447 RTLIB::SIN_PPCF128, Results);
4448 break;
4449 case ISD::FCOS:
4450 case ISD::STRICT_FCOS:
4451 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4452 RTLIB::COS_F80, RTLIB::COS_F128,
4453 RTLIB::COS_PPCF128, Results);
4454 break;
4455 case ISD::FSINCOS:
4456 // Expand into sincos libcall.
4457 ExpandSinCosLibCall(Node, Results);
4458 break;
4459 case ISD::FLOG:
4460 case ISD::STRICT_FLOG:
4461 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4462 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4463 break;
4464 case ISD::FLOG2:
4465 case ISD::STRICT_FLOG2:
4466 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4467 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4468 break;
4469 case ISD::FLOG10:
4470 case ISD::STRICT_FLOG10:
4471 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4472 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4473 break;
4474 case ISD::FEXP:
4475 case ISD::STRICT_FEXP:
4476 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4477 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4478 break;
4479 case ISD::FEXP2:
4480 case ISD::STRICT_FEXP2:
4481 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4482 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4483 break;
4484 case ISD::FEXP10:
4485 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4486 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);
4487 break;
4488 case ISD::FTRUNC:
4489 case ISD::STRICT_FTRUNC:
4490 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4491 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4492 RTLIB::TRUNC_PPCF128, Results);
4493 break;
4494 case ISD::FFLOOR:
4495 case ISD::STRICT_FFLOOR:
4496 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4497 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4498 RTLIB::FLOOR_PPCF128, Results);
4499 break;
4500 case ISD::FCEIL:
4501 case ISD::STRICT_FCEIL:
4502 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4503 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4504 RTLIB::CEIL_PPCF128, Results);
4505 break;
4506 case ISD::FRINT:
4507 case ISD::STRICT_FRINT:
4508 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4509 RTLIB::RINT_F80, RTLIB::RINT_F128,
4510 RTLIB::RINT_PPCF128, Results);
4511 break;
4512 case ISD::FNEARBYINT:
4514 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4515 RTLIB::NEARBYINT_F64,
4516 RTLIB::NEARBYINT_F80,
4517 RTLIB::NEARBYINT_F128,
4518 RTLIB::NEARBYINT_PPCF128, Results);
4519 break;
4520 case ISD::FROUND:
4521 case ISD::STRICT_FROUND:
4522 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4523 RTLIB::ROUND_F64,
4524 RTLIB::ROUND_F80,
4525 RTLIB::ROUND_F128,
4526 RTLIB::ROUND_PPCF128, Results);
4527 break;
4528 case ISD::FROUNDEVEN:
4530 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4531 RTLIB::ROUNDEVEN_F64,
4532 RTLIB::ROUNDEVEN_F80,
4533 RTLIB::ROUNDEVEN_F128,
4534 RTLIB::ROUNDEVEN_PPCF128, Results);
4535 break;
4536 case ISD::FLDEXP:
4537 case ISD::STRICT_FLDEXP:
4538 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4539 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
4540 break;
4541 case ISD::FFREXP: {
4542 ExpandFrexpLibCall(Node, Results);
4543 break;
4544 }
4545 case ISD::FPOWI:
4546 case ISD::STRICT_FPOWI: {
4547 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4548 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4549 if (!TLI.getLibcallName(LC)) {
4550 // Some targets don't have a powi libcall; use pow instead.
4551 if (Node->isStrictFPOpcode()) {
4553 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4554 {Node->getValueType(0), Node->getValueType(1)},
4555 {Node->getOperand(0), Node->getOperand(2)});
4556 SDValue FPOW =
4557 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4558 {Node->getValueType(0), Node->getValueType(1)},
4559 {Exponent.getValue(1), Node->getOperand(1), Exponent});
4560 Results.push_back(FPOW);
4561 Results.push_back(FPOW.getValue(1));
4562 } else {
4564 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4565 Node->getOperand(1));
4566 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4567 Node->getValueType(0),
4568 Node->getOperand(0), Exponent));
4569 }
4570 break;
4571 }
4572 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4573 bool ExponentHasSizeOfInt =
4574 DAG.getLibInfo().getIntSize() ==
4575 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4576 if (!ExponentHasSizeOfInt) {
4577 // If the exponent does not match with sizeof(int) a libcall to
4578 // RTLIB::POWI would use the wrong type for the argument.
4579 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4580 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4581 break;
4582 }
4583 ExpandFPLibCall(Node, LC, Results);
4584 break;
4585 }
4586 case ISD::FPOW:
4587 case ISD::STRICT_FPOW:
4588 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4589 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4590 break;
4591 case ISD::LROUND:
4592 case ISD::STRICT_LROUND:
4593 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4594 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4595 RTLIB::LROUND_F128,
4596 RTLIB::LROUND_PPCF128, Results);
4597 break;
4598 case ISD::LLROUND:
4600 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4601 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4602 RTLIB::LLROUND_F128,
4603 RTLIB::LLROUND_PPCF128, Results);
4604 break;
4605 case ISD::LRINT:
4606 case ISD::STRICT_LRINT:
4607 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4608 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4609 RTLIB::LRINT_F128,
4610 RTLIB::LRINT_PPCF128, Results);
4611 break;
4612 case ISD::LLRINT:
4613 case ISD::STRICT_LLRINT:
4614 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4615 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4616 RTLIB::LLRINT_F128,
4617 RTLIB::LLRINT_PPCF128, Results);
4618 break;
4619 case ISD::FDIV:
4620 case ISD::STRICT_FDIV:
4621 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4622 RTLIB::DIV_F80, RTLIB::DIV_F128,
4623 RTLIB::DIV_PPCF128, Results);
4624 break;
4625 case ISD::FREM:
4626 case ISD::STRICT_FREM:
4627 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4628 RTLIB::REM_F80, RTLIB::REM_F128,
4629 RTLIB::REM_PPCF128, Results);
4630 break;
4631 case ISD::FMA:
4632 case ISD::STRICT_FMA:
4633 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4634 RTLIB::FMA_F80, RTLIB::FMA_F128,
4635 RTLIB::FMA_PPCF128, Results);
4636 break;
4637 case ISD::FADD:
4638 case ISD::STRICT_FADD:
4639 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4640 RTLIB::ADD_F80, RTLIB::ADD_F128,
4641 RTLIB::ADD_PPCF128, Results);
4642 break;
4643 case ISD::FMUL:
4644 case ISD::STRICT_FMUL:
4645 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,