LLVM 19.0.0git
LegalizeTypes.h
Go to the documentation of this file.
1//===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the DAGTypeLegalizer class. This is a private interface
10// shared between the code that implements the SelectionDAG::LegalizeTypes
11// method.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
16#define LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
17
18#include "MatchContext.h"
19#include "llvm/ADT/DenseMap.h"
23
24namespace llvm {
25
26//===----------------------------------------------------------------------===//
27/// This takes an arbitrary SelectionDAG as input and hacks on it until only
28/// value types the target machine can handle are left. This involves promoting
29/// small sizes to large sizes or splitting up large values into small values.
30///
32 const TargetLowering &TLI;
33 SelectionDAG &DAG;
34public:
35 /// This pass uses the NodeId on the SDNodes to hold information about the
36 /// state of the node. The enum has all the values.
38 /// All operands have been processed, so this node is ready to be handled.
39 ReadyToProcess = 0,
40
41 /// This is a new node, not before seen, that was created in the process of
42 /// legalizing some other node.
43 NewNode = -1,
44
45 /// This node's ID needs to be set to the number of its unprocessed
46 /// operands.
47 Unanalyzed = -2,
48
49 /// This is a node that has already been processed.
50 Processed = -3
51
52 // 1+ - This is a node which has this many unprocessed operands.
53 };
54private:
55
56 /// This is a bitvector that contains two bits for each simple value type,
57 /// where the two bits correspond to the LegalizeAction enum from
58 /// TargetLowering. This can be queried with "getTypeAction(VT)".
60
61 /// Return how we should legalize values of this type.
62 TargetLowering::LegalizeTypeAction getTypeAction(EVT VT) const {
63 return TLI.getTypeAction(*DAG.getContext(), VT);
64 }
65
66 /// Return true if this type is legal on this target.
67 bool isTypeLegal(EVT VT) const {
68 return TLI.getTypeAction(*DAG.getContext(), VT) == TargetLowering::TypeLegal;
69 }
70
71 /// Return true if this is a simple legal type.
72 bool isSimpleLegalType(EVT VT) const {
73 return VT.isSimple() && TLI.isTypeLegal(VT);
74 }
75
76 EVT getSetCCResultType(EVT VT) const {
77 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
78 }
79
80 /// Pretend all of this node's results are legal.
81 bool IgnoreNodeResults(SDNode *N) const {
82 return N->getOpcode() == ISD::TargetConstant ||
83 N->getOpcode() == ISD::Register;
84 }
85
86 // Bijection from SDValue to unique id. As each created node gets a
87 // new id we do not need to worry about reuse expunging. Should we
88 // run out of ids, we can do a one time expensive compactifcation.
89 typedef unsigned TableId;
90
91 TableId NextValueId = 1;
92
95
96 /// For integer nodes that are below legal width, this map indicates what
97 /// promoted value to use.
99
100 /// For integer nodes that need to be expanded this map indicates which
101 /// operands are the expanded version of the input.
103
104 /// For floating-point nodes converted to integers of the same size, this map
105 /// indicates the converted value to use.
107
108 /// For floating-point nodes that have a smaller precision than the smallest
109 /// supported precision, this map indicates what promoted value to use.
111
112 /// For floating-point nodes that have a smaller precision than the smallest
113 /// supported precision, this map indicates the converted value to use.
114 SmallDenseMap<TableId, TableId, 8> SoftPromotedHalfs;
115
116 /// For float nodes that need to be expanded this map indicates which operands
117 /// are the expanded version of the input.
119
120 /// For nodes that are <1 x ty>, this map indicates the scalar value of type
121 /// 'ty' to use.
122 SmallDenseMap<TableId, TableId, 8> ScalarizedVectors;
123
124 /// For nodes that need to be split this map indicates which operands are the
125 /// expanded version of the input.
127
128 /// For vector nodes that need to be widened, indicates the widened value to
129 /// use.
131
132 /// For values that have been replaced with another, indicates the replacement
133 /// value to use.
135
136 /// This defines a worklist of nodes to process. In order to be pushed onto
137 /// this worklist, all operands of a node must have already been processed.
139
140 TableId getTableId(SDValue V) {
141 assert(V.getNode() && "Getting TableId on SDValue()");
142
143 auto I = ValueToIdMap.find(V);
144 if (I != ValueToIdMap.end()) {
145 // replace if there's been a shift.
146 RemapId(I->second);
147 assert(I->second && "All Ids should be nonzero");
148 return I->second;
149 }
150 // Add if it's not there.
151 ValueToIdMap.insert(std::make_pair(V, NextValueId));
152 IdToValueMap.insert(std::make_pair(NextValueId, V));
153 ++NextValueId;
154 assert(NextValueId != 0 &&
155 "Ran out of Ids. Increase id type size or add compactification");
156 return NextValueId - 1;
157 }
158
159 const SDValue &getSDValue(TableId &Id) {
160 RemapId(Id);
161 assert(Id && "TableId should be non-zero");
162 auto I = IdToValueMap.find(Id);
163 assert(I != IdToValueMap.end() && "cannot find Id in map");
164 return I->second;
165 }
166
167public:
169 : TLI(dag.getTargetLoweringInfo()), DAG(dag),
170 ValueTypeActions(TLI.getValueTypeActions()) {
171 }
172
173 /// This is the main entry point for the type legalizer. This does a
174 /// top-down traversal of the dag, legalizing types as it goes. Returns
175 /// "true" if it made any changes.
176 bool run();
177
178 void NoteDeletion(SDNode *Old, SDNode *New) {
179 assert(Old != New && "node replaced with self");
180 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
181 TableId NewId = getTableId(SDValue(New, i));
182 TableId OldId = getTableId(SDValue(Old, i));
183
184 if (OldId != NewId) {
185 ReplacedValues[OldId] = NewId;
186
187 // Delete Node from tables. We cannot do this when OldId == NewId,
188 // because NewId can still have table references to it in
189 // ReplacedValues.
190 IdToValueMap.erase(OldId);
191 PromotedIntegers.erase(OldId);
192 ExpandedIntegers.erase(OldId);
193 SoftenedFloats.erase(OldId);
194 PromotedFloats.erase(OldId);
195 SoftPromotedHalfs.erase(OldId);
196 ExpandedFloats.erase(OldId);
197 ScalarizedVectors.erase(OldId);
198 SplitVectors.erase(OldId);
199 WidenedVectors.erase(OldId);
200 }
201
202 ValueToIdMap.erase(SDValue(Old, i));
203 }
204 }
205
206 SelectionDAG &getDAG() const { return DAG; }
207
208private:
209 SDNode *AnalyzeNewNode(SDNode *N);
210 void AnalyzeNewValue(SDValue &Val);
211 void PerformExpensiveChecks();
212 void RemapId(TableId &Id);
213 void RemapValue(SDValue &V);
214
215 // Common routines.
216 SDValue BitConvertToInteger(SDValue Op);
217 SDValue BitConvertVectorToIntegerVector(SDValue Op);
218 SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
219 bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);
220 bool CustomWidenLowerNode(SDNode *N, EVT VT);
221
222 /// Replace each result of the given MERGE_VALUES node with the corresponding
223 /// input operand, except for the result 'ResNo', for which the corresponding
224 /// input operand is returned.
225 SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);
226
227 SDValue JoinIntegers(SDValue Lo, SDValue Hi);
228
229 std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
230
231 SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT);
232
233 void ReplaceValueWith(SDValue From, SDValue To);
234 void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
235 void SplitInteger(SDValue Op, EVT LoVT, EVT HiVT,
236 SDValue &Lo, SDValue &Hi);
237
238 //===--------------------------------------------------------------------===//
239 // Integer Promotion Support: LegalizeIntegerTypes.cpp
240 //===--------------------------------------------------------------------===//
241
242 /// Given a processed operand Op which was promoted to a larger integer type,
243 /// this returns the promoted value. The low bits of the promoted value
244 /// corresponding to the original type are exactly equal to Op.
245 /// The extra bits contain rubbish, so the promoted value may need to be zero-
246 /// or sign-extended from the original type before it is usable (the helpers
247 /// SExtPromotedInteger and ZExtPromotedInteger can do this for you).
248 /// For example, if Op is an i16 and was promoted to an i32, then this method
249 /// returns an i32, the lower 16 bits of which coincide with Op, and the upper
250 /// 16 bits of which contain rubbish.
251 SDValue GetPromotedInteger(SDValue Op) {
252 TableId &PromotedId = PromotedIntegers[getTableId(Op)];
253 SDValue PromotedOp = getSDValue(PromotedId);
254 assert(PromotedOp.getNode() && "Operand wasn't promoted?");
255 return PromotedOp;
256 }
257 void SetPromotedInteger(SDValue Op, SDValue Result);
258
259 /// Get a promoted operand and sign extend it to the final size.
260 SDValue SExtPromotedInteger(SDValue Op) {
261 EVT OldVT = Op.getValueType();
262 SDLoc dl(Op);
263 Op = GetPromotedInteger(Op);
264 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op,
265 DAG.getValueType(OldVT));
266 }
267
268 /// Get a promoted operand and zero extend it to the final size.
269 SDValue ZExtPromotedInteger(SDValue Op) {
270 EVT OldVT = Op.getValueType();
271 SDLoc dl(Op);
272 Op = GetPromotedInteger(Op);
273 return DAG.getZeroExtendInReg(Op, dl, OldVT);
274 }
275
276 /// Get a promoted operand and zero extend it to the final size.
277 SDValue VPSExtPromotedInteger(SDValue Op, SDValue Mask, SDValue EVL) {
278 EVT OldVT = Op.getValueType();
279 SDLoc dl(Op);
280 Op = GetPromotedInteger(Op);
281 // FIXME: Add VP_SIGN_EXTEND_INREG.
282 EVT VT = Op.getValueType();
283 unsigned BitsDiff = VT.getScalarSizeInBits() - OldVT.getScalarSizeInBits();
284 SDValue ShiftCst = DAG.getShiftAmountConstant(BitsDiff, VT, dl);
285 SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShiftCst, Mask, EVL);
286 return DAG.getNode(ISD::VP_SRA, dl, VT, Shl, ShiftCst, Mask, EVL);
287 }
288
289 /// Get a promoted operand and zero extend it to the final size.
290 SDValue VPZExtPromotedInteger(SDValue Op, SDValue Mask, SDValue EVL) {
291 EVT OldVT = Op.getValueType();
292 SDLoc dl(Op);
293 Op = GetPromotedInteger(Op);
294 return DAG.getVPZeroExtendInReg(Op, Mask, EVL, dl, OldVT);
295 }
296
297 // Promote the given operand V (vector or scalar) according to N's specific
298 // reduction kind. N must be an integer VECREDUCE_* or VP_REDUCE_*. Returns
299 // the nominal extension opcode (ISD::(ANY|ZERO|SIGN)_EXTEND) and the
300 // promoted value.
301 SDValue PromoteIntOpVectorReduction(SDNode *N, SDValue V);
302
303 // Integer Result Promotion.
304 void PromoteIntegerResult(SDNode *N, unsigned ResNo);
305 SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
306 SDValue PromoteIntRes_AssertSext(SDNode *N);
307 SDValue PromoteIntRes_AssertZext(SDNode *N);
308 SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);
309 SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);
310 SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo);
311 SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);
312 SDValue PromoteIntRes_INSERT_SUBVECTOR(SDNode *N);
313 SDValue PromoteIntRes_VECTOR_REVERSE(SDNode *N);
314 SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
315 SDValue PromoteIntRes_VECTOR_SPLICE(SDNode *N);
316 SDValue PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N);
317 SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
318 SDValue PromoteIntRes_ScalarOp(SDNode *N);
319 SDValue PromoteIntRes_STEP_VECTOR(SDNode *N);
320 SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N);
321 SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);
322 SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);
323 SDValue PromoteIntRes_BITCAST(SDNode *N);
324 SDValue PromoteIntRes_BSWAP(SDNode *N);
325 SDValue PromoteIntRes_BITREVERSE(SDNode *N);
326 SDValue PromoteIntRes_BUILD_PAIR(SDNode *N);
327 SDValue PromoteIntRes_Constant(SDNode *N);
328 SDValue PromoteIntRes_CTLZ(SDNode *N);
329 SDValue PromoteIntRes_CTPOP_PARITY(SDNode *N);
330 SDValue PromoteIntRes_CTTZ(SDNode *N);
331 SDValue PromoteIntRes_VP_CttzElements(SDNode *N);
332 SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N);
333 SDValue PromoteIntRes_FP_TO_XINT(SDNode *N);
334 SDValue PromoteIntRes_FP_TO_XINT_SAT(SDNode *N);
335 SDValue PromoteIntRes_FP_TO_FP16_BF16(SDNode *N);
336 SDValue PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N);
337 SDValue PromoteIntRes_XRINT(SDNode *N);
338 SDValue PromoteIntRes_FREEZE(SDNode *N);
339 SDValue PromoteIntRes_INT_EXTEND(SDNode *N);
340 SDValue PromoteIntRes_LOAD(LoadSDNode *N);
341 SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
342 SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
343 SDValue PromoteIntRes_Overflow(SDNode *N);
344 SDValue PromoteIntRes_FFREXP(SDNode *N);
345 SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
346 SDValue PromoteIntRes_CMP(SDNode *N);
347 SDValue PromoteIntRes_Select(SDNode *N);
348 SDValue PromoteIntRes_SELECT_CC(SDNode *N);
349 SDValue PromoteIntRes_SETCC(SDNode *N);
350 SDValue PromoteIntRes_SHL(SDNode *N);
351 SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N);
352 SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N);
353 SDValue PromoteIntRes_SExtIntBinOp(SDNode *N);
354 SDValue PromoteIntRes_UMINUMAX(SDNode *N);
355 SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N);
356 SDValue PromoteIntRes_SRA(SDNode *N);
357 SDValue PromoteIntRes_SRL(SDNode *N);
358 SDValue PromoteIntRes_TRUNCATE(SDNode *N);
359 SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);
360 SDValue PromoteIntRes_UADDSUBO_CARRY(SDNode *N, unsigned ResNo);
361 SDValue PromoteIntRes_SADDSUBO_CARRY(SDNode *N, unsigned ResNo);
362 SDValue PromoteIntRes_UNDEF(SDNode *N);
363 SDValue PromoteIntRes_VAARG(SDNode *N);
364 SDValue PromoteIntRes_VSCALE(SDNode *N);
365 SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
366 template <class MatchContextClass>
367 SDValue PromoteIntRes_ADDSUBSHLSAT(SDNode *N);
368 SDValue PromoteIntRes_MULFIX(SDNode *N);
369 SDValue PromoteIntRes_DIVFIX(SDNode *N);
370 SDValue PromoteIntRes_GET_ROUNDING(SDNode *N);
371 SDValue PromoteIntRes_VECREDUCE(SDNode *N);
372 SDValue PromoteIntRes_VP_REDUCE(SDNode *N);
373 SDValue PromoteIntRes_ABS(SDNode *N);
374 SDValue PromoteIntRes_Rotate(SDNode *N);
375 SDValue PromoteIntRes_FunnelShift(SDNode *N);
376 SDValue PromoteIntRes_VPFunnelShift(SDNode *N);
377 SDValue PromoteIntRes_IS_FPCLASS(SDNode *N);
378 SDValue PromoteIntRes_PATCHPOINT(SDNode *N);
379
380 // Integer Operand Promotion.
381 bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);
382 SDValue PromoteIntOp_ANY_EXTEND(SDNode *N);
383 SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N);
384 SDValue PromoteIntOp_BITCAST(SDNode *N);
385 SDValue PromoteIntOp_BUILD_PAIR(SDNode *N);
386 SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo);
387 SDValue PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo);
388 SDValue PromoteIntOp_BUILD_VECTOR(SDNode *N);
389 SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo);
390 SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N);
391 SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N);
392 SDValue PromoteIntOp_INSERT_SUBVECTOR(SDNode *N);
393 SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
394 SDValue PromoteIntOp_ScalarOp(SDNode *N);
395 SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
396 SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
397 SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
398 SDValue PromoteIntOp_Shift(SDNode *N);
399 SDValue PromoteIntOp_CMP(SDNode *N);
400 SDValue PromoteIntOp_FunnelShift(SDNode *N);
401 SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);
402 SDValue PromoteIntOp_VP_SIGN_EXTEND(SDNode *N);
403 SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);
404 SDValue PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N);
405 SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo);
406 SDValue PromoteIntOp_TRUNCATE(SDNode *N);
407 SDValue PromoteIntOp_UINT_TO_FP(SDNode *N);
408 SDValue PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N);
409 SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N);
410 SDValue PromoteIntOp_VP_ZERO_EXTEND(SDNode *N);
411 SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
412 SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
413 SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
414 SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
415 SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
416 SDValue PromoteIntOp_FIX(SDNode *N);
417 SDValue PromoteIntOp_ExpOp(SDNode *N);
418 SDValue PromoteIntOp_VECREDUCE(SDNode *N);
419 SDValue PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo);
420 SDValue PromoteIntOp_SET_ROUNDING(SDNode *N);
421 SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo);
422 SDValue PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo);
423 SDValue PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo);
424 SDValue PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo);
425
426 void SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS);
427 void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
428
429 //===--------------------------------------------------------------------===//
430 // Integer Expansion Support: LegalizeIntegerTypes.cpp
431 //===--------------------------------------------------------------------===//
432
433 /// Given a processed operand Op which was expanded into two integers of half
434 /// the size, this returns the two halves. The low bits of Op are exactly
435 /// equal to the bits of Lo; the high bits exactly equal Hi.
436 /// For example, if Op is an i64 which was expanded into two i32's, then this
437 /// method returns the two i32's, with Lo being equal to the lower 32 bits of
438 /// Op, and Hi being equal to the upper 32 bits.
439 void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
440 void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi);
441
442 // Integer Result Expansion.
443 void ExpandIntegerResult(SDNode *N, unsigned ResNo);
444 void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
445 void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi);
446 void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);
447 void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi);
448 void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi);
449 void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);
450 void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);
451 void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);
452 void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi);
453 void ExpandIntRes_READCOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi);
454 void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
455 void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi);
456 void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi);
457 void ExpandIntRes_ZERO_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
458 void ExpandIntRes_GET_ROUNDING (SDNode *N, SDValue &Lo, SDValue &Hi);
459 void ExpandIntRes_FP_TO_XINT (SDNode *N, SDValue &Lo, SDValue &Hi);
460 void ExpandIntRes_FP_TO_XINT_SAT (SDNode *N, SDValue &Lo, SDValue &Hi);
461 void ExpandIntRes_XROUND_XRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
462
463 void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi);
464 void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
465 void ExpandIntRes_ADDSUBC (SDNode *N, SDValue &Lo, SDValue &Hi);
466 void ExpandIntRes_ADDSUBE (SDNode *N, SDValue &Lo, SDValue &Hi);
467 void ExpandIntRes_UADDSUBO_CARRY (SDNode *N, SDValue &Lo, SDValue &Hi);
468 void ExpandIntRes_SADDSUBO_CARRY (SDNode *N, SDValue &Lo, SDValue &Hi);
469 void ExpandIntRes_BITREVERSE (SDNode *N, SDValue &Lo, SDValue &Hi);
470 void ExpandIntRes_BSWAP (SDNode *N, SDValue &Lo, SDValue &Hi);
471 void ExpandIntRes_PARITY (SDNode *N, SDValue &Lo, SDValue &Hi);
472 void ExpandIntRes_MUL (SDNode *N, SDValue &Lo, SDValue &Hi);
473 void ExpandIntRes_SDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
474 void ExpandIntRes_SREM (SDNode *N, SDValue &Lo, SDValue &Hi);
475 void ExpandIntRes_UDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
476 void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi);
477 void ExpandIntRes_ShiftThroughStack (SDNode *N, SDValue &Lo, SDValue &Hi);
478 void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi);
479
480 void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi);
481
482 void ExpandIntRes_CMP (SDNode *N, SDValue &Lo, SDValue &Hi);
483
484 void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
485 void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
486 void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi);
487 void ExpandIntRes_AVG (SDNode *N, SDValue &Lo, SDValue &Hi);
488 void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi);
489 void ExpandIntRes_SHLSAT (SDNode *N, SDValue &Lo, SDValue &Hi);
490 void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi);
491 void ExpandIntRes_DIVFIX (SDNode *N, SDValue &Lo, SDValue &Hi);
492
493 void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
494 void ExpandIntRes_VECREDUCE (SDNode *N, SDValue &Lo, SDValue &Hi);
495
496 void ExpandIntRes_Rotate (SDNode *N, SDValue &Lo, SDValue &Hi);
497 void ExpandIntRes_FunnelShift (SDNode *N, SDValue &Lo, SDValue &Hi);
498
499 void ExpandIntRes_VSCALE (SDNode *N, SDValue &Lo, SDValue &Hi);
500
501 void ExpandShiftByConstant(SDNode *N, const APInt &Amt,
502 SDValue &Lo, SDValue &Hi);
503 bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
504 bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
505
506 // Integer Operand Expansion.
507 bool ExpandIntegerOperand(SDNode *N, unsigned OpNo);
508 SDValue ExpandIntOp_BR_CC(SDNode *N);
509 SDValue ExpandIntOp_SELECT_CC(SDNode *N);
510 SDValue ExpandIntOp_SETCC(SDNode *N);
511 SDValue ExpandIntOp_SETCCCARRY(SDNode *N);
512 SDValue ExpandIntOp_Shift(SDNode *N);
513 SDValue ExpandIntOp_CMP(SDNode *N);
514 SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);
515 SDValue ExpandIntOp_TRUNCATE(SDNode *N);
516 SDValue ExpandIntOp_XINT_TO_FP(SDNode *N);
517 SDValue ExpandIntOp_RETURNADDR(SDNode *N);
518 SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
519 SDValue ExpandIntOp_SPLAT_VECTOR(SDNode *N);
520 SDValue ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo);
521 SDValue ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo);
522 SDValue ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo);
523
524 void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
525 ISD::CondCode &CCCode, const SDLoc &dl);
526
527 //===--------------------------------------------------------------------===//
528 // Float to Integer Conversion Support: LegalizeFloatTypes.cpp
529 //===--------------------------------------------------------------------===//
530
531 /// GetSoftenedFloat - Given a processed operand Op which was converted to an
532 /// integer of the same size, this returns the integer. The integer contains
533 /// exactly the same bits as Op - only the type changed. For example, if Op
534 /// is an f32 which was softened to an i32, then this method returns an i32,
535 /// the bits of which coincide with those of Op
536 SDValue GetSoftenedFloat(SDValue Op) {
537 TableId Id = getTableId(Op);
538 auto Iter = SoftenedFloats.find(Id);
539 if (Iter == SoftenedFloats.end()) {
540 assert(isSimpleLegalType(Op.getValueType()) &&
541 "Operand wasn't converted to integer?");
542 return Op;
543 }
544 SDValue SoftenedOp = getSDValue(Iter->second);
545 assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?");
546 return SoftenedOp;
547 }
548 void SetSoftenedFloat(SDValue Op, SDValue Result);
549
550 // Convert Float Results to Integer.
551 void SoftenFloatResult(SDNode *N, unsigned ResNo);
552 SDValue SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC);
553 SDValue SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC);
554 SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
555 SDValue SoftenFloatRes_ARITH_FENCE(SDNode *N);
556 SDValue SoftenFloatRes_BITCAST(SDNode *N);
557 SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N);
558 SDValue SoftenFloatRes_ConstantFP(SDNode *N);
559 SDValue SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N);
560 SDValue SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo);
561 SDValue SoftenFloatRes_FABS(SDNode *N);
562 SDValue SoftenFloatRes_FACOS(SDNode *N);
563 SDValue SoftenFloatRes_FASIN(SDNode *N);
564 SDValue SoftenFloatRes_FATAN(SDNode *N);
565 SDValue SoftenFloatRes_FMINNUM(SDNode *N);
566 SDValue SoftenFloatRes_FMAXNUM(SDNode *N);
567 SDValue SoftenFloatRes_FADD(SDNode *N);
568 SDValue SoftenFloatRes_FCBRT(SDNode *N);
569 SDValue SoftenFloatRes_FCEIL(SDNode *N);
570 SDValue SoftenFloatRes_FCOPYSIGN(SDNode *N);
571 SDValue SoftenFloatRes_FCOS(SDNode *N);
572 SDValue SoftenFloatRes_FCOSH(SDNode *N);
573 SDValue SoftenFloatRes_FDIV(SDNode *N);
574 SDValue SoftenFloatRes_FEXP(SDNode *N);
575 SDValue SoftenFloatRes_FEXP2(SDNode *N);
576 SDValue SoftenFloatRes_FEXP10(SDNode *N);
577 SDValue SoftenFloatRes_FFLOOR(SDNode *N);
578 SDValue SoftenFloatRes_FLOG(SDNode *N);
579 SDValue SoftenFloatRes_FLOG2(SDNode *N);
580 SDValue SoftenFloatRes_FLOG10(SDNode *N);
581 SDValue SoftenFloatRes_FMA(SDNode *N);
582 SDValue SoftenFloatRes_FMUL(SDNode *N);
583 SDValue SoftenFloatRes_FNEARBYINT(SDNode *N);
584 SDValue SoftenFloatRes_FNEG(SDNode *N);
585 SDValue SoftenFloatRes_FP_EXTEND(SDNode *N);
586 SDValue SoftenFloatRes_FP16_TO_FP(SDNode *N);
587 SDValue SoftenFloatRes_BF16_TO_FP(SDNode *N);
588 SDValue SoftenFloatRes_FP_ROUND(SDNode *N);
589 SDValue SoftenFloatRes_FPOW(SDNode *N);
590 SDValue SoftenFloatRes_ExpOp(SDNode *N);
591 SDValue SoftenFloatRes_FFREXP(SDNode *N);
592 SDValue SoftenFloatRes_FREEZE(SDNode *N);
593 SDValue SoftenFloatRes_FREM(SDNode *N);
594 SDValue SoftenFloatRes_FRINT(SDNode *N);
595 SDValue SoftenFloatRes_FROUND(SDNode *N);
596 SDValue SoftenFloatRes_FROUNDEVEN(SDNode *N);
597 SDValue SoftenFloatRes_FSIN(SDNode *N);
598 SDValue SoftenFloatRes_FSINH(SDNode *N);
599 SDValue SoftenFloatRes_FSQRT(SDNode *N);
600 SDValue SoftenFloatRes_FSUB(SDNode *N);
601 SDValue SoftenFloatRes_FTAN(SDNode *N);
602 SDValue SoftenFloatRes_FTANH(SDNode *N);
603 SDValue SoftenFloatRes_FTRUNC(SDNode *N);
604 SDValue SoftenFloatRes_LOAD(SDNode *N);
605 SDValue SoftenFloatRes_ATOMIC_LOAD(SDNode *N);
606 SDValue SoftenFloatRes_SELECT(SDNode *N);
607 SDValue SoftenFloatRes_SELECT_CC(SDNode *N);
608 SDValue SoftenFloatRes_UNDEF(SDNode *N);
609 SDValue SoftenFloatRes_VAARG(SDNode *N);
610 SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
611 SDValue SoftenFloatRes_VECREDUCE(SDNode *N);
612 SDValue SoftenFloatRes_VECREDUCE_SEQ(SDNode *N);
613
614 // Convert Float Operand to Integer.
615 bool SoftenFloatOperand(SDNode *N, unsigned OpNo);
616 SDValue SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC);
617 SDValue SoftenFloatOp_BITCAST(SDNode *N);
618 SDValue SoftenFloatOp_BR_CC(SDNode *N);
619 SDValue SoftenFloatOp_FP_ROUND(SDNode *N);
620 SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N);
621 SDValue SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N);
622 SDValue SoftenFloatOp_LROUND(SDNode *N);
623 SDValue SoftenFloatOp_LLROUND(SDNode *N);
624 SDValue SoftenFloatOp_LRINT(SDNode *N);
625 SDValue SoftenFloatOp_LLRINT(SDNode *N);
626 SDValue SoftenFloatOp_SELECT_CC(SDNode *N);
627 SDValue SoftenFloatOp_SETCC(SDNode *N);
628 SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo);
629 SDValue SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
630 SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N);
631
632 //===--------------------------------------------------------------------===//
633 // Float Expansion Support: LegalizeFloatTypes.cpp
634 //===--------------------------------------------------------------------===//
635
636 /// Given a processed operand Op which was expanded into two floating-point
637 /// values of half the size, this returns the two halves.
638 /// The low bits of Op are exactly equal to the bits of Lo; the high bits
639 /// exactly equal Hi. For example, if Op is a ppcf128 which was expanded
640 /// into two f64's, then this method returns the two f64's, with Lo being
641 /// equal to the lower 64 bits of Op, and Hi to the upper 64 bits.
642 void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi);
643 void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi);
644
645 // Float Result Expansion.
646 void ExpandFloatResult(SDNode *N, unsigned ResNo);
647 void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi);
648 void ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
649 SDValue &Lo, SDValue &Hi);
650 void ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
651 SDValue &Lo, SDValue &Hi);
652 // clang-format off
653 void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi);
654 void ExpandFloatRes_FACOS (SDNode *N, SDValue &Lo, SDValue &Hi);
655 void ExpandFloatRes_FASIN (SDNode *N, SDValue &Lo, SDValue &Hi);
656 void ExpandFloatRes_FATAN (SDNode *N, SDValue &Lo, SDValue &Hi);
657 void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
658 void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
659 void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi);
660 void ExpandFloatRes_FCBRT (SDNode *N, SDValue &Lo, SDValue &Hi);
661 void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi);
662 void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi);
663 void ExpandFloatRes_FCOS (SDNode *N, SDValue &Lo, SDValue &Hi);
664 void ExpandFloatRes_FCOSH (SDNode *N, SDValue &Lo, SDValue &Hi);
665 void ExpandFloatRes_FDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
666 void ExpandFloatRes_FEXP (SDNode *N, SDValue &Lo, SDValue &Hi);
667 void ExpandFloatRes_FEXP2 (SDNode *N, SDValue &Lo, SDValue &Hi);
668 void ExpandFloatRes_FEXP10 (SDNode *N, SDValue &Lo, SDValue &Hi);
669 void ExpandFloatRes_FFLOOR (SDNode *N, SDValue &Lo, SDValue &Hi);
670 void ExpandFloatRes_FLOG (SDNode *N, SDValue &Lo, SDValue &Hi);
671 void ExpandFloatRes_FLOG2 (SDNode *N, SDValue &Lo, SDValue &Hi);
672 void ExpandFloatRes_FLOG10 (SDNode *N, SDValue &Lo, SDValue &Hi);
673 void ExpandFloatRes_FMA (SDNode *N, SDValue &Lo, SDValue &Hi);
674 void ExpandFloatRes_FMUL (SDNode *N, SDValue &Lo, SDValue &Hi);
675 void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi);
676 void ExpandFloatRes_FNEG (SDNode *N, SDValue &Lo, SDValue &Hi);
677 void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
678 void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi);
679 void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi);
680 void ExpandFloatRes_FLDEXP (SDNode *N, SDValue &Lo, SDValue &Hi);
681 void ExpandFloatRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);
682 void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi);
683 void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
684 void ExpandFloatRes_FROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
685 void ExpandFloatRes_FROUNDEVEN(SDNode *N, SDValue &Lo, SDValue &Hi);
686 void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi);
687 void ExpandFloatRes_FSINH (SDNode *N, SDValue &Lo, SDValue &Hi);
688 void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi);
689 void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
690 void ExpandFloatRes_FTAN (SDNode *N, SDValue &Lo, SDValue &Hi);
691 void ExpandFloatRes_FTANH (SDNode *N, SDValue &Lo, SDValue &Hi);
692 void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi);
693 void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
694 void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);
695 // clang-format on
696
697 // Float Operand Expansion.
698 bool ExpandFloatOperand(SDNode *N, unsigned OpNo);
699 SDValue ExpandFloatOp_BR_CC(SDNode *N);
700 SDValue ExpandFloatOp_FCOPYSIGN(SDNode *N);
701 SDValue ExpandFloatOp_FP_ROUND(SDNode *N);
702 SDValue ExpandFloatOp_FP_TO_XINT(SDNode *N);
703 SDValue ExpandFloatOp_LROUND(SDNode *N);
704 SDValue ExpandFloatOp_LLROUND(SDNode *N);
705 SDValue ExpandFloatOp_LRINT(SDNode *N);
706 SDValue ExpandFloatOp_LLRINT(SDNode *N);
707 SDValue ExpandFloatOp_SELECT_CC(SDNode *N);
708 SDValue ExpandFloatOp_SETCC(SDNode *N);
709 SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo);
710
711 void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
712 ISD::CondCode &CCCode, const SDLoc &dl,
713 SDValue &Chain, bool IsSignaling = false);
714
715 //===--------------------------------------------------------------------===//
716 // Float promotion support: LegalizeFloatTypes.cpp
717 //===--------------------------------------------------------------------===//
718
719 SDValue GetPromotedFloat(SDValue Op) {
720 TableId &PromotedId = PromotedFloats[getTableId(Op)];
721 SDValue PromotedOp = getSDValue(PromotedId);
722 assert(PromotedOp.getNode() && "Operand wasn't promoted?");
723 return PromotedOp;
724 }
725 void SetPromotedFloat(SDValue Op, SDValue Result);
726
727 void PromoteFloatResult(SDNode *N, unsigned ResNo);
728 SDValue PromoteFloatRes_BITCAST(SDNode *N);
729 SDValue PromoteFloatRes_BinOp(SDNode *N);
730 SDValue PromoteFloatRes_ConstantFP(SDNode *N);
731 SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N);
732 SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N);
733 SDValue PromoteFloatRes_FMAD(SDNode *N);
734 SDValue PromoteFloatRes_ExpOp(SDNode *N);
735 SDValue PromoteFloatRes_FFREXP(SDNode *N);
736 SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
737 SDValue PromoteFloatRes_STRICT_FP_ROUND(SDNode *N);
738 SDValue PromoteFloatRes_LOAD(SDNode *N);
739 SDValue PromoteFloatRes_ATOMIC_LOAD(SDNode *N);
740 SDValue PromoteFloatRes_SELECT(SDNode *N);
741 SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
742 SDValue PromoteFloatRes_UnaryOp(SDNode *N);
743 SDValue PromoteFloatRes_UNDEF(SDNode *N);
744 SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);
745 SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
746 SDValue PromoteFloatRes_VECREDUCE(SDNode *N);
747 SDValue PromoteFloatRes_VECREDUCE_SEQ(SDNode *N);
748
749 bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
750 SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
751 SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
752 SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
753 SDValue PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, unsigned OpNo);
754 SDValue PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo);
755 SDValue PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, unsigned OpNo);
756 SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
757 SDValue PromoteFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
758 SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
759 SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
760
761 //===--------------------------------------------------------------------===//
762 // Half soft promotion support: LegalizeFloatTypes.cpp
763 //===--------------------------------------------------------------------===//
764
765 SDValue GetSoftPromotedHalf(SDValue Op) {
766 TableId &PromotedId = SoftPromotedHalfs[getTableId(Op)];
767 SDValue PromotedOp = getSDValue(PromotedId);
768 assert(PromotedOp.getNode() && "Operand wasn't promoted?");
769 return PromotedOp;
770 }
771 void SetSoftPromotedHalf(SDValue Op, SDValue Result);
772
773 void SoftPromoteHalfResult(SDNode *N, unsigned ResNo);
774 SDValue SoftPromoteHalfRes_ARITH_FENCE(SDNode *N);
775 SDValue SoftPromoteHalfRes_BinOp(SDNode *N);
776 SDValue SoftPromoteHalfRes_BITCAST(SDNode *N);
777 SDValue SoftPromoteHalfRes_ConstantFP(SDNode *N);
778 SDValue SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N);
779 SDValue SoftPromoteHalfRes_FCOPYSIGN(SDNode *N);
780 SDValue SoftPromoteHalfRes_FMAD(SDNode *N);
781 SDValue SoftPromoteHalfRes_ExpOp(SDNode *N);
782 SDValue SoftPromoteHalfRes_FFREXP(SDNode *N);
783 SDValue SoftPromoteHalfRes_FP_ROUND(SDNode *N);
784 SDValue SoftPromoteHalfRes_LOAD(SDNode *N);
785 SDValue SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N);
786 SDValue SoftPromoteHalfRes_SELECT(SDNode *N);
787 SDValue SoftPromoteHalfRes_SELECT_CC(SDNode *N);
788 SDValue SoftPromoteHalfRes_UnaryOp(SDNode *N);
789 SDValue SoftPromoteHalfRes_XINT_TO_FP(SDNode *N);
790 SDValue SoftPromoteHalfRes_UNDEF(SDNode *N);
791 SDValue SoftPromoteHalfRes_VECREDUCE(SDNode *N);
792 SDValue SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N);
793
794 bool SoftPromoteHalfOperand(SDNode *N, unsigned OpNo);
795 SDValue SoftPromoteHalfOp_BITCAST(SDNode *N);
796 SDValue SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
797 SDValue SoftPromoteHalfOp_FP_EXTEND(SDNode *N);
798 SDValue SoftPromoteHalfOp_FP_TO_XINT(SDNode *N);
799 SDValue SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N);
800 SDValue SoftPromoteHalfOp_SETCC(SDNode *N);
801 SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo);
802 SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo);
803 SDValue SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
804 SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo);
805 SDValue SoftPromoteHalfOp_PATCHPOINT(SDNode *N, unsigned OpNo);
806
807 //===--------------------------------------------------------------------===//
808 // Scalarization Support: LegalizeVectorTypes.cpp
809 //===--------------------------------------------------------------------===//
810
811 /// Given a processed one-element vector Op which was scalarized to its
812 /// element type, this returns the element. For example, if Op is a v1i32,
813 /// Op = < i32 val >, this method returns val, an i32.
814 SDValue GetScalarizedVector(SDValue Op) {
815 TableId &ScalarizedId = ScalarizedVectors[getTableId(Op)];
816 SDValue ScalarizedOp = getSDValue(ScalarizedId);
817 assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?");
818 return ScalarizedOp;
819 }
820 void SetScalarizedVector(SDValue Op, SDValue Result);
821
822 // Vector Result Scalarization: <1 x ty> -> ty.
823 void ScalarizeVectorResult(SDNode *N, unsigned ResNo);
824 SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
825 SDValue ScalarizeVecRes_BinOp(SDNode *N);
826 SDValue ScalarizeVecRes_CMP(SDNode *N);
827 SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
828 SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
829 SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
830 SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
831 SDValue ScalarizeVecRes_InregOp(SDNode *N);
832 SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
833
834 SDValue ScalarizeVecRes_ADDRSPACECAST(SDNode *N);
835 SDValue ScalarizeVecRes_BITCAST(SDNode *N);
836 SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
837 SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
838 SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
839 SDValue ScalarizeVecRes_ExpOp(SDNode *N);
840 SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N);
841 SDValue ScalarizeVecRes_LOAD(LoadSDNode *N);
842 SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
843 SDValue ScalarizeVecRes_VSELECT(SDNode *N);
844 SDValue ScalarizeVecRes_SELECT(SDNode *N);
845 SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
846 SDValue ScalarizeVecRes_SETCC(SDNode *N);
847 SDValue ScalarizeVecRes_UNDEF(SDNode *N);
848 SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N);
849 SDValue ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N);
850 SDValue ScalarizeVecRes_IS_FPCLASS(SDNode *N);
851
852 SDValue ScalarizeVecRes_FIX(SDNode *N);
853 SDValue ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo);
854
855 // Vector Operand Scalarization: <1 x ty> -> ty.
856 bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo);
857 SDValue ScalarizeVecOp_BITCAST(SDNode *N);
858 SDValue ScalarizeVecOp_UnaryOp(SDNode *N);
859 SDValue ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N);
860 SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);
861 SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
862 SDValue ScalarizeVecOp_VSELECT(SDNode *N);
863 SDValue ScalarizeVecOp_VSETCC(SDNode *N);
864 SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo);
865 SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo);
866 SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo);
867 SDValue ScalarizeVecOp_FP_EXTEND(SDNode *N);
868 SDValue ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N);
869 SDValue ScalarizeVecOp_VECREDUCE(SDNode *N);
870 SDValue ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N);
871 SDValue ScalarizeVecOp_CMP(SDNode *N);
872
873 //===--------------------------------------------------------------------===//
874 // Vector Splitting Support: LegalizeVectorTypes.cpp
875 //===--------------------------------------------------------------------===//
876
877 /// Given a processed vector Op which was split into vectors of half the size,
878 /// this method returns the halves. The first elements of Op coincide with the
879 /// elements of Lo; the remaining elements of Op coincide with the elements of
880 /// Hi: Op is what you would get by concatenating Lo and Hi.
881 /// For example, if Op is a v8i32 that was split into two v4i32's, then this
882 /// method returns the two v4i32's, with Lo corresponding to the first 4
883 /// elements of Op, and Hi to the last 4 elements.
884 void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi);
885 void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi);
886
887 /// Split mask operator of a VP intrinsic.
888 std::pair<SDValue, SDValue> SplitMask(SDValue Mask);
889
890 /// Split mask operator of a VP intrinsic in a given location.
891 std::pair<SDValue, SDValue> SplitMask(SDValue Mask, const SDLoc &DL);
892
893 // Helper function for incrementing the pointer when splitting
894 // memory operations
895 void IncrementPointer(MemSDNode *N, EVT MemVT, MachinePointerInfo &MPI,
896 SDValue &Ptr, uint64_t *ScaledOffset = nullptr);
897
898 // Vector Result Splitting: <128 x ty> -> 2 x <64 x ty>.
899 void SplitVectorResult(SDNode *N, unsigned ResNo);
900 void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);
901 void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
902 void SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi);
903 void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
904 void SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo, SDValue &Hi);
905 void SplitVecRes_FFREXP(SDNode *N, unsigned ResNo, SDValue &Lo, SDValue &Hi);
906 void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
907 void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
908 void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
909 void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
910 void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
911 SDValue &Lo, SDValue &Hi);
912
913 void SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi);
914
915 void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
916 void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
917 void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);
918 void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
919 void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
920 void SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo, SDValue &Hi);
921 void SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo, SDValue &Hi);
922 void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
923 void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);
924 void SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo, SDValue &Hi);
925 void SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD, SDValue &Lo,
926 SDValue &Hi);
927 void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
928 void SplitVecRes_Gather(MemSDNode *VPGT, SDValue &Lo, SDValue &Hi,
929 bool SplitSETCC = false);
930 void SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi);
931 void SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
932 void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
933 void SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);
934 void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
935 SDValue &Hi);
936 void SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo, SDValue &Hi);
937 void SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N);
938 void SplitVecRes_VECTOR_INTERLEAVE(SDNode *N);
939 void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi);
940 void SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, SDValue &Hi);
941 void SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);
942
943 // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>.
944 bool SplitVectorOperand(SDNode *N, unsigned OpNo);
945 SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
946 SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);
947 SDValue SplitVecOp_VECREDUCE_SEQ(SDNode *N);
948 SDValue SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo);
949 SDValue SplitVecOp_UnaryOp(SDNode *N);
950 SDValue SplitVecOp_TruncateHelper(SDNode *N);
951
952 SDValue SplitVecOp_BITCAST(SDNode *N);
953 SDValue SplitVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo);
954 SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);
955 SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
956 SDValue SplitVecOp_ExtVecInRegOp(SDNode *N);
957 SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);
958 SDValue SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo);
959 SDValue SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N, unsigned OpNo);
960 SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
961 SDValue SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo);
962 SDValue SplitVecOp_Gather(MemSDNode *MGT, unsigned OpNo);
963 SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);
964 SDValue SplitVecOp_VSETCC(SDNode *N);
965 SDValue SplitVecOp_FP_ROUND(SDNode *N);
966 SDValue SplitVecOp_FPOpDifferentTypes(SDNode *N);
967 SDValue SplitVecOp_CMP(SDNode *N);
968 SDValue SplitVecOp_FP_TO_XINT_SAT(SDNode *N);
969 SDValue SplitVecOp_VP_CttzElements(SDNode *N);
970
971 //===--------------------------------------------------------------------===//
972 // Vector Widening Support: LegalizeVectorTypes.cpp
973 //===--------------------------------------------------------------------===//
974
975 /// Given a processed vector Op which was widened into a larger vector, this
976 /// method returns the larger vector. The elements of the returned vector
977 /// consist of the elements of Op followed by elements containing rubbish.
978 /// For example, if Op is a v2i32 that was widened to a v4i32, then this
979 /// method returns a v4i32 for which the first two elements are the same as
980 /// those of Op, while the last two elements contain rubbish.
981 SDValue GetWidenedVector(SDValue Op) {
982 TableId &WidenedId = WidenedVectors[getTableId(Op)];
983 SDValue WidenedOp = getSDValue(WidenedId);
984 assert(WidenedOp.getNode() && "Operand wasn't widened?");
985 return WidenedOp;
986 }
987 void SetWidenedVector(SDValue Op, SDValue Result);
988
989 /// Given a mask Mask, returns the larger vector into which Mask was widened.
990 SDValue GetWidenedMask(SDValue Mask, ElementCount EC) {
991 // For VP operations, we must also widen the mask. Note that the mask type
992 // may not actually need widening, leading it be split along with the VP
993 // operation.
994 // FIXME: This could lead to an infinite split/widen loop. We only handle
995 // the case where the mask needs widening to an identically-sized type as
996 // the vector inputs.
997 assert(getTypeAction(Mask.getValueType()) ==
998 TargetLowering::TypeWidenVector &&
999 "Unable to widen binary VP op");
1000 Mask = GetWidenedVector(Mask);
1001 assert(Mask.getValueType().getVectorElementCount() == EC &&
1002 "Unable to widen binary VP op");
1003 return Mask;
1004 }
1005
1006 // Widen Vector Result Promotion.
1007 void WidenVectorResult(SDNode *N, unsigned ResNo);
1008 SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
1009 SDValue WidenVecRes_ADDRSPACECAST(SDNode *N);
1010 SDValue WidenVecRes_AssertZext(SDNode* N);
1011 SDValue WidenVecRes_BITCAST(SDNode* N);
1012 SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
1013 SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
1014 SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N);
1015 SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
1016 SDValue WidenVecRes_INSERT_SUBVECTOR(SDNode *N);
1017 SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
1018 SDValue WidenVecRes_LOAD(SDNode* N);
1019 SDValue WidenVecRes_VP_LOAD(VPLoadSDNode *N);
1020 SDValue WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N);
1021 SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
1022 SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
1023 SDValue WidenVecRes_VP_GATHER(VPGatherSDNode* N);
1024 SDValue WidenVecRes_ScalarOp(SDNode* N);
1025 SDValue WidenVecRes_Select(SDNode *N);
1026 SDValue WidenVSELECTMask(SDNode *N);
1027 SDValue WidenVecRes_SELECT_CC(SDNode* N);
1028 SDValue WidenVecRes_SETCC(SDNode* N);
1029 SDValue WidenVecRes_STRICT_FSETCC(SDNode* N);
1030 SDValue WidenVecRes_UNDEF(SDNode *N);
1031 SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
1032 SDValue WidenVecRes_VECTOR_REVERSE(SDNode *N);
1033
1034 SDValue WidenVecRes_Ternary(SDNode *N);
1035 SDValue WidenVecRes_Binary(SDNode *N);
1036 SDValue WidenVecRes_CMP(SDNode *N);
1037 SDValue WidenVecRes_BinaryCanTrap(SDNode *N);
1038 SDValue WidenVecRes_BinaryWithExtraScalarOp(SDNode *N);
1039 SDValue WidenVecRes_StrictFP(SDNode *N);
1040 SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo);
1041 SDValue WidenVecRes_Convert(SDNode *N);
1042 SDValue WidenVecRes_Convert_StrictFP(SDNode *N);
1043 SDValue WidenVecRes_FP_TO_XINT_SAT(SDNode *N);
1044 SDValue WidenVecRes_XRINT(SDNode *N);
1045 SDValue WidenVecRes_FCOPYSIGN(SDNode *N);
1046 SDValue WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N);
1047 SDValue WidenVecRes_ExpOp(SDNode *N);
1048 SDValue WidenVecRes_Unary(SDNode *N);
1049 SDValue WidenVecRes_InregOp(SDNode *N);
1050
1051 // Widen Vector Operand.
1052 bool WidenVectorOperand(SDNode *N, unsigned OpNo);
1053 SDValue WidenVecOp_BITCAST(SDNode *N);
1054 SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
1055 SDValue WidenVecOp_EXTEND(SDNode *N);
1056 SDValue WidenVecOp_CMP(SDNode *N);
1057 SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
1058 SDValue WidenVecOp_INSERT_SUBVECTOR(SDNode *N);
1059 SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
1060 SDValue WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N);
1061 SDValue WidenVecOp_STORE(SDNode* N);
1062 SDValue WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo);
1063 SDValue WidenVecOp_VP_STRIDED_STORE(SDNode *N, unsigned OpNo);
1064 SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
1065 SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
1066 SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
1067 SDValue WidenVecOp_VP_SCATTER(SDNode* N, unsigned OpNo);
1068 SDValue WidenVecOp_SETCC(SDNode* N);
1069 SDValue WidenVecOp_STRICT_FSETCC(SDNode* N);
1070 SDValue WidenVecOp_VSELECT(SDNode *N);
1071
1072 SDValue WidenVecOp_Convert(SDNode *N);
1073 SDValue WidenVecOp_FP_TO_XINT_SAT(SDNode *N);
1074 SDValue WidenVecOp_UnrollVectorOp(SDNode *N);
1075 SDValue WidenVecOp_IS_FPCLASS(SDNode *N);
1076 SDValue WidenVecOp_VECREDUCE(SDNode *N);
1077 SDValue WidenVecOp_VECREDUCE_SEQ(SDNode *N);
1078 SDValue WidenVecOp_VP_REDUCE(SDNode *N);
1079 SDValue WidenVecOp_ExpOp(SDNode *N);
1080 SDValue WidenVecOp_VP_CttzElements(SDNode *N);
1081
1082 /// Helper function to generate a set of operations to perform
1083 /// a vector operation for a wider type.
1084 ///
1085 SDValue UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE);
1086
1087 //===--------------------------------------------------------------------===//
1088 // Vector Widening Utilities Support: LegalizeVectorTypes.cpp
1089 //===--------------------------------------------------------------------===//
1090
1091 /// Helper function to generate a set of loads to load a vector with a
1092 /// resulting wider type. It takes:
1093 /// LdChain: list of chains for the load to be generated.
1094 /// Ld: load to widen
1095 SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
1096 LoadSDNode *LD);
1097
1098 /// Helper function to generate a set of extension loads to load a vector with
1099 /// a resulting wider type. It takes:
1100 /// LdChain: list of chains for the load to be generated.
1101 /// Ld: load to widen
1102 /// ExtType: extension element type
1103 SDValue GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
1104 LoadSDNode *LD, ISD::LoadExtType ExtType);
1105
1106 /// Helper function to generate a set of stores to store a widen vector into
1107 /// non-widen memory. Returns true if successful, false otherwise.
1108 /// StChain: list of chains for the stores we have generated
1109 /// ST: store of a widen value
1110 bool GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, StoreSDNode *ST);
1111
1112 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
1113 /// input vector must have the same element type as NVT.
1114 /// When FillWithZeroes is "on" the vector will be widened with zeroes.
1115 /// By default, the vector will be widened with undefined values.
1116 SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false);
1117
1118 /// Return a mask of vector type MaskVT to replace InMask. Also adjust
1119 /// MaskVT to ToMaskVT if needed with vector extension or truncation.
1120 SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT);
1121
1122 //===--------------------------------------------------------------------===//
1123 // Generic Splitting: LegalizeTypesGeneric.cpp
1124 //===--------------------------------------------------------------------===//
1125
1126 // Legalization methods which only use that the illegal type is split into two
1127 // not necessarily identical types. As such they can be used for splitting
1128 // vectors and expanding integers and floats.
1129
1130 void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
1131 if (Op.getValueType().isVector())
1132 GetSplitVector(Op, Lo, Hi);
1133 else if (Op.getValueType().isInteger())
1134 GetExpandedInteger(Op, Lo, Hi);
1135 else
1136 GetExpandedFloat(Op, Lo, Hi);
1137 }
1138
1139 /// Use ISD::EXTRACT_ELEMENT nodes to extract the low and high parts of the
1140 /// given value.
1141 void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
1142
1143 // Generic Result Splitting.
1144 void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
1145 SDValue &Lo, SDValue &Hi);
1146 void SplitVecRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);
1147 void SplitRes_ARITH_FENCE (SDNode *N, SDValue &Lo, SDValue &Hi);
1148 void SplitRes_Select (SDNode *N, SDValue &Lo, SDValue &Hi);
1149 void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
1150 void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
1151 void SplitRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);
1152
1153 //===--------------------------------------------------------------------===//
1154 // Generic Expansion: LegalizeTypesGeneric.cpp
1155 //===--------------------------------------------------------------------===//
1156
1157 // Legalization methods which only use that the illegal type is split into two
1158 // identical types of half the size, and that the Lo/Hi part is stored first
1159 // in memory on little/big-endian machines, followed by the Hi/Lo part. As
1160 // such they can be used for expanding integers and floats.
1161
1162 void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
1163 if (Op.getValueType().isInteger())
1164 GetExpandedInteger(Op, Lo, Hi);
1165 else
1166 GetExpandedFloat(Op, Lo, Hi);
1167 }
1168
1169
1170 /// This function will split the integer \p Op into \p NumElements
1171 /// operations of type \p EltVT and store them in \p Ops.
1172 void IntegerToVector(SDValue Op, unsigned NumElements,
1173 SmallVectorImpl<SDValue> &Ops, EVT EltVT);
1174
1175 // Generic Result Expansion.
1176 void ExpandRes_MERGE_VALUES (SDNode *N, unsigned ResNo,
1177 SDValue &Lo, SDValue &Hi);
1178 void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi);
1179 void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi);
1180 void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi);
1181 void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
1182 void ExpandRes_NormalLoad (SDNode *N, SDValue &Lo, SDValue &Hi);
1183 void ExpandRes_VAARG (SDNode *N, SDValue &Lo, SDValue &Hi);
1184
1185 // Generic Operand Expansion.
1186 SDValue ExpandOp_BITCAST (SDNode *N);
1187 SDValue ExpandOp_BUILD_VECTOR (SDNode *N);
1188 SDValue ExpandOp_EXTRACT_ELEMENT (SDNode *N);
1189 SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N);
1190 SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N);
1191 SDValue ExpandOp_NormalStore (SDNode *N, unsigned OpNo);
1192};
1193
1194} // end namespace llvm.
1195
1196#endif
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
BlockVerifier::State From
#define LLVM_LIBRARY_VISIBILITY
Definition: Compiler.h:131
This file defines the DenseMap class.
#define I(x, y, z)
Definition: MD5.cpp:58
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
Class for arbitrary precision integers.
Definition: APInt.h:78
This is an SDNode representing atomic operations.
This takes an arbitrary SelectionDAG as input and hacks on it until only value types the target machi...
Definition: LegalizeTypes.h:31
DAGTypeLegalizer(SelectionDAG &dag)
void NoteDeletion(SDNode *Old, SDNode *New)
SelectionDAG & getDAG() const
NodeIdFlags
This pass uses the NodeId on the SDNodes to hold information about the state of the node.
Definition: LegalizeTypes.h:37
This class represents an Operation in the Expression.
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
bool erase(const KeyT &Val)
Definition: DenseMap.h:345
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
This class is used to represent ISD::LOAD nodes.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
This is an abstract virtual class for memory operations.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:227
SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:486
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVMContext * getContext() const
Definition: SelectionDAG.h:499
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
LegalizeTypeAction getTypeAction(MVT VT) const
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:818
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:164
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1566
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1546
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
DWARFExpression::Operation Op
#define N
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
This class contains a discriminated union of information about pointers in memory operands,...