LLVM 19.0.0git
HexagonISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- HexagonISelDAGToDAG.cpp - A dag to dag inst selector for Hexagon --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the Hexagon target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "HexagonISelDAGToDAG.h"
14#include "Hexagon.h"
15#include "HexagonISelLowering.h"
21#include "llvm/IR/Intrinsics.h"
22#include "llvm/IR/IntrinsicsHexagon.h"
24#include "llvm/Support/Debug.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "hexagon-isel"
28#define PASS_NAME "Hexagon DAG->DAG Pattern Instruction Selection"
29
30static
32EnableAddressRebalancing("isel-rebalance-addr", cl::Hidden, cl::init(true),
33 cl::desc("Rebalance address calculation trees to improve "
34 "instruction selection"));
35
36// Rebalance only if this allows e.g. combining a GA with an offset or
37// factoring out a shift.
38static
41 cl::desc("Rebalance address tree only if this allows optimizations"));
42
43static
46 cl::init(false), cl::desc("Rebalance address tree only if it is imbalanced"));
47
48static cl::opt<bool> CheckSingleUse("hexagon-isel-su", cl::Hidden,
49 cl::init(true), cl::desc("Enable checking of SDNode's single-use status"));
50
51//===----------------------------------------------------------------------===//
52// Instruction Selector Implementation
53//===----------------------------------------------------------------------===//
54
55#define GET_DAGISEL_BODY HexagonDAGToDAGISel
56#include "HexagonGenDAGISel.inc"
57
58namespace llvm {
59/// createHexagonISelDag - This pass converts a legalized DAG into a
60/// Hexagon-specific DAG, ready for instruction scheduling.
62 CodeGenOptLevel OptLevel) {
63 return new HexagonDAGToDAGISel(TM, OptLevel);
64}
65}
66
68
70
71void HexagonDAGToDAGISel::SelectIndexedLoad(LoadSDNode *LD, const SDLoc &dl) {
72 SDValue Chain = LD->getChain();
73 SDValue Base = LD->getBasePtr();
74 SDValue Offset = LD->getOffset();
75 int32_t Inc = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
76 EVT LoadedVT = LD->getMemoryVT();
77 unsigned Opcode = 0;
78
79 // Check for zero extended loads. Treat any-extend loads as zero extended
80 // loads.
81 ISD::LoadExtType ExtType = LD->getExtensionType();
82 bool IsZeroExt = (ExtType == ISD::ZEXTLOAD || ExtType == ISD::EXTLOAD);
83 bool IsValidInc = HII->isValidAutoIncImm(LoadedVT, Inc);
84
85 assert(LoadedVT.isSimple());
86 switch (LoadedVT.getSimpleVT().SimpleTy) {
87 case MVT::i8:
88 if (IsZeroExt)
89 Opcode = IsValidInc ? Hexagon::L2_loadrub_pi : Hexagon::L2_loadrub_io;
90 else
91 Opcode = IsValidInc ? Hexagon::L2_loadrb_pi : Hexagon::L2_loadrb_io;
92 break;
93 case MVT::i16:
94 if (IsZeroExt)
95 Opcode = IsValidInc ? Hexagon::L2_loadruh_pi : Hexagon::L2_loadruh_io;
96 else
97 Opcode = IsValidInc ? Hexagon::L2_loadrh_pi : Hexagon::L2_loadrh_io;
98 break;
99 case MVT::i32:
100 case MVT::f32:
101 case MVT::v2i16:
102 case MVT::v4i8:
103 Opcode = IsValidInc ? Hexagon::L2_loadri_pi : Hexagon::L2_loadri_io;
104 break;
105 case MVT::i64:
106 case MVT::f64:
107 case MVT::v2i32:
108 case MVT::v4i16:
109 case MVT::v8i8:
110 Opcode = IsValidInc ? Hexagon::L2_loadrd_pi : Hexagon::L2_loadrd_io;
111 break;
112 case MVT::v64i8:
113 case MVT::v32i16:
114 case MVT::v16i32:
115 case MVT::v8i64:
116 case MVT::v128i8:
117 case MVT::v64i16:
118 case MVT::v32i32:
119 case MVT::v16i64:
120 if (isAlignedMemNode(LD)) {
121 if (LD->isNonTemporal())
122 Opcode = IsValidInc ? Hexagon::V6_vL32b_nt_pi : Hexagon::V6_vL32b_nt_ai;
123 else
124 Opcode = IsValidInc ? Hexagon::V6_vL32b_pi : Hexagon::V6_vL32b_ai;
125 } else {
126 Opcode = IsValidInc ? Hexagon::V6_vL32Ub_pi : Hexagon::V6_vL32Ub_ai;
127 }
128 break;
129 default:
130 llvm_unreachable("Unexpected memory type in indexed load");
131 }
132
133 SDValue IncV = CurDAG->getTargetConstant(Inc, dl, MVT::i32);
134 MachineMemOperand *MemOp = LD->getMemOperand();
135
136 auto getExt64 = [this,ExtType] (MachineSDNode *N, const SDLoc &dl)
137 -> MachineSDNode* {
138 if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::EXTLOAD) {
139 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
140 return CurDAG->getMachineNode(Hexagon::A4_combineir, dl, MVT::i64,
141 Zero, SDValue(N, 0));
142 }
143 if (ExtType == ISD::SEXTLOAD)
144 return CurDAG->getMachineNode(Hexagon::A2_sxtw, dl, MVT::i64,
145 SDValue(N, 0));
146 return N;
147 };
148
149 // Loaded value Next address Chain
150 SDValue From[3] = { SDValue(LD,0), SDValue(LD,1), SDValue(LD,2) };
151 SDValue To[3];
152
153 EVT ValueVT = LD->getValueType(0);
154 if (ValueVT == MVT::i64 && ExtType != ISD::NON_EXTLOAD) {
155 // A load extending to i64 will actually produce i32, which will then
156 // need to be extended to i64.
157 assert(LoadedVT.getSizeInBits() <= 32);
158 ValueVT = MVT::i32;
159 }
160
161 if (IsValidInc) {
162 MachineSDNode *L = CurDAG->getMachineNode(Opcode, dl, ValueVT,
163 MVT::i32, MVT::Other, Base,
164 IncV, Chain);
165 CurDAG->setNodeMemRefs(L, {MemOp});
166 To[1] = SDValue(L, 1); // Next address.
167 To[2] = SDValue(L, 2); // Chain.
168 // Handle special case for extension to i64.
169 if (LD->getValueType(0) == MVT::i64)
170 L = getExt64(L, dl);
171 To[0] = SDValue(L, 0); // Loaded (extended) value.
172 } else {
173 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
174 MachineSDNode *L = CurDAG->getMachineNode(Opcode, dl, ValueVT, MVT::Other,
175 Base, Zero, Chain);
176 CurDAG->setNodeMemRefs(L, {MemOp});
177 To[2] = SDValue(L, 1); // Chain.
178 MachineSDNode *A = CurDAG->getMachineNode(Hexagon::A2_addi, dl, MVT::i32,
179 Base, IncV);
180 To[1] = SDValue(A, 0); // Next address.
181 // Handle special case for extension to i64.
182 if (LD->getValueType(0) == MVT::i64)
183 L = getExt64(L, dl);
184 To[0] = SDValue(L, 0); // Loaded (extended) value.
185 }
186 ReplaceUses(From, To, 3);
187 CurDAG->RemoveDeadNode(LD);
188}
189
191 if (IntN->getOpcode() != ISD::INTRINSIC_W_CHAIN)
192 return nullptr;
193
194 SDLoc dl(IntN);
195 unsigned IntNo = IntN->getConstantOperandVal(1);
196
197 static std::map<unsigned,unsigned> LoadPciMap = {
198 { Intrinsic::hexagon_circ_ldb, Hexagon::L2_loadrb_pci },
199 { Intrinsic::hexagon_circ_ldub, Hexagon::L2_loadrub_pci },
200 { Intrinsic::hexagon_circ_ldh, Hexagon::L2_loadrh_pci },
201 { Intrinsic::hexagon_circ_lduh, Hexagon::L2_loadruh_pci },
202 { Intrinsic::hexagon_circ_ldw, Hexagon::L2_loadri_pci },
203 { Intrinsic::hexagon_circ_ldd, Hexagon::L2_loadrd_pci },
204 };
205 auto FLC = LoadPciMap.find(IntNo);
206 if (FLC != LoadPciMap.end()) {
207 EVT ValTy = (IntNo == Intrinsic::hexagon_circ_ldd) ? MVT::i64 : MVT::i32;
208 EVT RTys[] = { ValTy, MVT::i32, MVT::Other };
209 // Operands: { Base, Increment, Modifier, Chain }
210 auto Inc = cast<ConstantSDNode>(IntN->getOperand(5));
211 SDValue I = CurDAG->getTargetConstant(Inc->getSExtValue(), dl, MVT::i32);
212 MachineSDNode *Res = CurDAG->getMachineNode(FLC->second, dl, RTys,
213 { IntN->getOperand(2), I, IntN->getOperand(4),
214 IntN->getOperand(0) });
215 return Res;
216 }
217
218 return nullptr;
219}
220
222 SDNode *IntN) {
223 // The "LoadN" is just a machine load instruction. The intrinsic also
224 // involves storing it. Generate an appropriate store to the location
225 // given in the intrinsic's operand(3).
226 uint64_t F = HII->get(LoadN->getMachineOpcode()).TSFlags;
227 unsigned SizeBits = (F >> HexagonII::MemAccessSizePos) &
229 unsigned Size = 1U << (SizeBits-1);
230
231 SDLoc dl(IntN);
233 SDValue TS;
234 SDValue Loc = IntN->getOperand(3);
235
236 if (Size >= 4)
237 TS = CurDAG->getStore(SDValue(LoadN, 2), dl, SDValue(LoadN, 0), Loc, PI,
238 Align(Size));
239 else
240 TS = CurDAG->getTruncStore(SDValue(LoadN, 2), dl, SDValue(LoadN, 0), Loc,
241 PI, MVT::getIntegerVT(Size * 8), Align(Size));
242
243 SDNode *StoreN;
244 {
245 HandleSDNode Handle(TS);
246 SelectStore(TS.getNode());
247 StoreN = Handle.getValue().getNode();
248 }
249
250 // Load's results are { Loaded value, Updated pointer, Chain }
251 ReplaceUses(SDValue(IntN, 0), SDValue(LoadN, 1));
252 ReplaceUses(SDValue(IntN, 1), SDValue(StoreN, 0));
253 return StoreN;
254}
255
257 // The intrinsics for load circ/brev perform two operations:
258 // 1. Load a value V from the specified location, using the addressing
259 // mode corresponding to the intrinsic.
260 // 2. Store V into a specified location. This location is typically a
261 // local, temporary object.
262 // In many cases, the program using these intrinsics will immediately
263 // load V again from the local object. In those cases, when certain
264 // conditions are met, the last load can be removed.
265 // This function identifies and optimizes this pattern. If the pattern
266 // cannot be optimized, it returns nullptr, which will cause the load
267 // to be selected separately from the intrinsic (which will be handled
268 // in SelectIntrinsicWChain).
269
270 SDValue Ch = N->getOperand(0);
271 SDValue Loc = N->getOperand(1);
272
273 // Assume that the load and the intrinsic are connected directly with a
274 // chain:
275 // t1: i32,ch = int.load ..., ..., ..., Loc, ... // <-- C
276 // t2: i32,ch = load t1:1, Loc, ...
277 SDNode *C = Ch.getNode();
278
279 if (C->getOpcode() != ISD::INTRINSIC_W_CHAIN)
280 return false;
281
282 // The second load can only be eliminated if its extension type matches
283 // that of the load instruction corresponding to the intrinsic. The user
284 // can provide an address of an unsigned variable to store the result of
285 // a sign-extending intrinsic into (or the other way around).
286 ISD::LoadExtType IntExt;
287 switch (C->getConstantOperandVal(1)) {
288 case Intrinsic::hexagon_circ_ldub:
289 case Intrinsic::hexagon_circ_lduh:
290 IntExt = ISD::ZEXTLOAD;
291 break;
292 case Intrinsic::hexagon_circ_ldw:
293 case Intrinsic::hexagon_circ_ldd:
294 IntExt = ISD::NON_EXTLOAD;
295 break;
296 default:
297 IntExt = ISD::SEXTLOAD;
298 break;
299 }
300 if (N->getExtensionType() != IntExt)
301 return false;
302
303 // Make sure the target location for the loaded value in the load intrinsic
304 // is the location from which LD (or N) is loading.
305 if (C->getNumOperands() < 4 || Loc.getNode() != C->getOperand(3).getNode())
306 return false;
307
310 SDValue F[] = { SDValue(N,0), SDValue(N,1), SDValue(C,0), SDValue(C,1) };
311 SDValue T[] = { SDValue(L,0), SDValue(S,0), SDValue(L,1), SDValue(S,0) };
312 ReplaceUses(F, T, std::size(T));
313 // This transformation will leave the intrinsic dead. If it remains in
314 // the DAG, the selection code will see it again, but without the load,
315 // and it will generate a store that is normally required for it.
317 return true;
318 }
319 return false;
320}
321
322// Convert the bit-reverse load intrinsic to appropriate target instruction.
324 if (IntN->getOpcode() != ISD::INTRINSIC_W_CHAIN)
325 return false;
326
327 const SDLoc &dl(IntN);
328 unsigned IntNo = IntN->getConstantOperandVal(1);
329
330 static const std::map<unsigned, unsigned> LoadBrevMap = {
331 { Intrinsic::hexagon_L2_loadrb_pbr, Hexagon::L2_loadrb_pbr },
332 { Intrinsic::hexagon_L2_loadrub_pbr, Hexagon::L2_loadrub_pbr },
333 { Intrinsic::hexagon_L2_loadrh_pbr, Hexagon::L2_loadrh_pbr },
334 { Intrinsic::hexagon_L2_loadruh_pbr, Hexagon::L2_loadruh_pbr },
335 { Intrinsic::hexagon_L2_loadri_pbr, Hexagon::L2_loadri_pbr },
336 { Intrinsic::hexagon_L2_loadrd_pbr, Hexagon::L2_loadrd_pbr }
337 };
338 auto FLI = LoadBrevMap.find(IntNo);
339 if (FLI != LoadBrevMap.end()) {
340 EVT ValTy =
341 (IntNo == Intrinsic::hexagon_L2_loadrd_pbr) ? MVT::i64 : MVT::i32;
342 EVT RTys[] = { ValTy, MVT::i32, MVT::Other };
343 // Operands of Intrinsic: {chain, enum ID of intrinsic, baseptr,
344 // modifier}.
345 // Operands of target instruction: { Base, Modifier, Chain }.
347 FLI->second, dl, RTys,
348 {IntN->getOperand(2), IntN->getOperand(3), IntN->getOperand(0)});
349
350 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(IntN)->getMemOperand();
351 CurDAG->setNodeMemRefs(Res, {MemOp});
352
353 ReplaceUses(SDValue(IntN, 0), SDValue(Res, 0));
354 ReplaceUses(SDValue(IntN, 1), SDValue(Res, 1));
355 ReplaceUses(SDValue(IntN, 2), SDValue(Res, 2));
356 CurDAG->RemoveDeadNode(IntN);
357 return true;
358 }
359 return false;
360}
361
362/// Generate a machine instruction node for the new circular buffer intrinsics.
363/// The new versions use a CSx register instead of the K field.
365 if (IntN->getOpcode() != ISD::INTRINSIC_W_CHAIN)
366 return false;
367
368 SDLoc DL(IntN);
369 unsigned IntNo = IntN->getConstantOperandVal(1);
371
372 static std::map<unsigned,unsigned> LoadNPcMap = {
373 { Intrinsic::hexagon_L2_loadrub_pci, Hexagon::PS_loadrub_pci },
374 { Intrinsic::hexagon_L2_loadrb_pci, Hexagon::PS_loadrb_pci },
375 { Intrinsic::hexagon_L2_loadruh_pci, Hexagon::PS_loadruh_pci },
376 { Intrinsic::hexagon_L2_loadrh_pci, Hexagon::PS_loadrh_pci },
377 { Intrinsic::hexagon_L2_loadri_pci, Hexagon::PS_loadri_pci },
378 { Intrinsic::hexagon_L2_loadrd_pci, Hexagon::PS_loadrd_pci },
379 { Intrinsic::hexagon_L2_loadrub_pcr, Hexagon::PS_loadrub_pcr },
380 { Intrinsic::hexagon_L2_loadrb_pcr, Hexagon::PS_loadrb_pcr },
381 { Intrinsic::hexagon_L2_loadruh_pcr, Hexagon::PS_loadruh_pcr },
382 { Intrinsic::hexagon_L2_loadrh_pcr, Hexagon::PS_loadrh_pcr },
383 { Intrinsic::hexagon_L2_loadri_pcr, Hexagon::PS_loadri_pcr },
384 { Intrinsic::hexagon_L2_loadrd_pcr, Hexagon::PS_loadrd_pcr }
385 };
386 auto FLI = LoadNPcMap.find (IntNo);
387 if (FLI != LoadNPcMap.end()) {
388 EVT ValTy = MVT::i32;
389 if (IntNo == Intrinsic::hexagon_L2_loadrd_pci ||
390 IntNo == Intrinsic::hexagon_L2_loadrd_pcr)
391 ValTy = MVT::i64;
392 EVT RTys[] = { ValTy, MVT::i32, MVT::Other };
393 // Handle load.*_pci case which has 6 operands.
394 if (IntN->getNumOperands() == 6) {
395 auto Inc = cast<ConstantSDNode>(IntN->getOperand(3));
396 SDValue I = CurDAG->getTargetConstant(Inc->getSExtValue(), DL, MVT::i32);
397 // Operands: { Base, Increment, Modifier, Start, Chain }.
398 Ops = { IntN->getOperand(2), I, IntN->getOperand(4), IntN->getOperand(5),
399 IntN->getOperand(0) };
400 } else
401 // Handle load.*_pcr case which has 5 operands.
402 // Operands: { Base, Modifier, Start, Chain }.
403 Ops = { IntN->getOperand(2), IntN->getOperand(3), IntN->getOperand(4),
404 IntN->getOperand(0) };
405 MachineSDNode *Res = CurDAG->getMachineNode(FLI->second, DL, RTys, Ops);
406 ReplaceUses(SDValue(IntN, 0), SDValue(Res, 0));
407 ReplaceUses(SDValue(IntN, 1), SDValue(Res, 1));
408 ReplaceUses(SDValue(IntN, 2), SDValue(Res, 2));
409 CurDAG->RemoveDeadNode(IntN);
410 return true;
411 }
412
413 static std::map<unsigned,unsigned> StoreNPcMap = {
414 { Intrinsic::hexagon_S2_storerb_pci, Hexagon::PS_storerb_pci },
415 { Intrinsic::hexagon_S2_storerh_pci, Hexagon::PS_storerh_pci },
416 { Intrinsic::hexagon_S2_storerf_pci, Hexagon::PS_storerf_pci },
417 { Intrinsic::hexagon_S2_storeri_pci, Hexagon::PS_storeri_pci },
418 { Intrinsic::hexagon_S2_storerd_pci, Hexagon::PS_storerd_pci },
419 { Intrinsic::hexagon_S2_storerb_pcr, Hexagon::PS_storerb_pcr },
420 { Intrinsic::hexagon_S2_storerh_pcr, Hexagon::PS_storerh_pcr },
421 { Intrinsic::hexagon_S2_storerf_pcr, Hexagon::PS_storerf_pcr },
422 { Intrinsic::hexagon_S2_storeri_pcr, Hexagon::PS_storeri_pcr },
423 { Intrinsic::hexagon_S2_storerd_pcr, Hexagon::PS_storerd_pcr }
424 };
425 auto FSI = StoreNPcMap.find (IntNo);
426 if (FSI != StoreNPcMap.end()) {
427 EVT RTys[] = { MVT::i32, MVT::Other };
428 // Handle store.*_pci case which has 7 operands.
429 if (IntN->getNumOperands() == 7) {
430 auto Inc = cast<ConstantSDNode>(IntN->getOperand(3));
431 SDValue I = CurDAG->getTargetConstant(Inc->getSExtValue(), DL, MVT::i32);
432 // Operands: { Base, Increment, Modifier, Value, Start, Chain }.
433 Ops = { IntN->getOperand(2), I, IntN->getOperand(4), IntN->getOperand(5),
434 IntN->getOperand(6), IntN->getOperand(0) };
435 } else
436 // Handle store.*_pcr case which has 6 operands.
437 // Operands: { Base, Modifier, Value, Start, Chain }.
438 Ops = { IntN->getOperand(2), IntN->getOperand(3), IntN->getOperand(4),
439 IntN->getOperand(5), IntN->getOperand(0) };
440 MachineSDNode *Res = CurDAG->getMachineNode(FSI->second, DL, RTys, Ops);
441 ReplaceUses(SDValue(IntN, 0), SDValue(Res, 0));
442 ReplaceUses(SDValue(IntN, 1), SDValue(Res, 1));
443 CurDAG->RemoveDeadNode(IntN);
444 return true;
445 }
446
447 return false;
448}
449
451 SDLoc dl(N);
452 LoadSDNode *LD = cast<LoadSDNode>(N);
453
454 // Handle indexed loads.
455 ISD::MemIndexedMode AM = LD->getAddressingMode();
456 if (AM != ISD::UNINDEXED) {
457 SelectIndexedLoad(LD, dl);
458 return;
459 }
460
461 // Handle patterns using circ/brev load intrinsics.
463 return;
464
465 SelectCode(LD);
466}
467
469 SDValue Chain = ST->getChain();
470 SDValue Base = ST->getBasePtr();
471 SDValue Offset = ST->getOffset();
472 SDValue Value = ST->getValue();
473 // Get the constant value.
474 int32_t Inc = cast<ConstantSDNode>(Offset.getNode())->getSExtValue();
475 EVT StoredVT = ST->getMemoryVT();
476 EVT ValueVT = Value.getValueType();
477
478 bool IsValidInc = HII->isValidAutoIncImm(StoredVT, Inc);
479 unsigned Opcode = 0;
480
481 assert(StoredVT.isSimple());
482 switch (StoredVT.getSimpleVT().SimpleTy) {
483 case MVT::i8:
484 Opcode = IsValidInc ? Hexagon::S2_storerb_pi : Hexagon::S2_storerb_io;
485 break;
486 case MVT::i16:
487 Opcode = IsValidInc ? Hexagon::S2_storerh_pi : Hexagon::S2_storerh_io;
488 break;
489 case MVT::i32:
490 case MVT::f32:
491 case MVT::v2i16:
492 case MVT::v4i8:
493 Opcode = IsValidInc ? Hexagon::S2_storeri_pi : Hexagon::S2_storeri_io;
494 break;
495 case MVT::i64:
496 case MVT::f64:
497 case MVT::v2i32:
498 case MVT::v4i16:
499 case MVT::v8i8:
500 Opcode = IsValidInc ? Hexagon::S2_storerd_pi : Hexagon::S2_storerd_io;
501 break;
502 case MVT::v64i8:
503 case MVT::v32i16:
504 case MVT::v16i32:
505 case MVT::v8i64:
506 case MVT::v128i8:
507 case MVT::v64i16:
508 case MVT::v32i32:
509 case MVT::v16i64:
510 if (isAlignedMemNode(ST)) {
511 if (ST->isNonTemporal())
512 Opcode = IsValidInc ? Hexagon::V6_vS32b_nt_pi : Hexagon::V6_vS32b_nt_ai;
513 else
514 Opcode = IsValidInc ? Hexagon::V6_vS32b_pi : Hexagon::V6_vS32b_ai;
515 } else {
516 Opcode = IsValidInc ? Hexagon::V6_vS32Ub_pi : Hexagon::V6_vS32Ub_ai;
517 }
518 break;
519 default:
520 llvm_unreachable("Unexpected memory type in indexed store");
521 }
522
523 if (ST->isTruncatingStore() && ValueVT.getSizeInBits() == 64) {
524 assert(StoredVT.getSizeInBits() < 64 && "Not a truncating store");
525 Value = CurDAG->getTargetExtractSubreg(Hexagon::isub_lo,
526 dl, MVT::i32, Value);
527 }
528
529 SDValue IncV = CurDAG->getTargetConstant(Inc, dl, MVT::i32);
530 MachineMemOperand *MemOp = ST->getMemOperand();
531
532 // Next address Chain
533 SDValue From[2] = { SDValue(ST,0), SDValue(ST,1) };
534 SDValue To[2];
535
536 if (IsValidInc) {
537 // Build post increment store.
538 SDValue Ops[] = { Base, IncV, Value, Chain };
539 MachineSDNode *S = CurDAG->getMachineNode(Opcode, dl, MVT::i32, MVT::Other,
540 Ops);
542 To[0] = SDValue(S, 0);
543 To[1] = SDValue(S, 1);
544 } else {
545 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
546 SDValue Ops[] = { Base, Zero, Value, Chain };
547 MachineSDNode *S = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
549 To[1] = SDValue(S, 0);
550 MachineSDNode *A = CurDAG->getMachineNode(Hexagon::A2_addi, dl, MVT::i32,
551 Base, IncV);
552 To[0] = SDValue(A, 0);
553 }
554
555 ReplaceUses(From, To, 2);
557}
558
560 SDLoc dl(N);
561 StoreSDNode *ST = cast<StoreSDNode>(N);
562
563 // Handle indexed stores.
564 ISD::MemIndexedMode AM = ST->getAddressingMode();
565 if (AM != ISD::UNINDEXED) {
566 SelectIndexedStore(ST, dl);
567 return;
568 }
569
570 SelectCode(ST);
571}
572
574 SDLoc dl(N);
575 SDValue Shl_0 = N->getOperand(0);
576 SDValue Shl_1 = N->getOperand(1);
577
578 auto Default = [this,N] () -> void { SelectCode(N); };
579
580 if (N->getValueType(0) != MVT::i32 || Shl_1.getOpcode() != ISD::Constant)
581 return Default();
582
583 // RHS is const.
584 int32_t ShlConst = cast<ConstantSDNode>(Shl_1)->getSExtValue();
585
586 if (Shl_0.getOpcode() == ISD::MUL) {
587 SDValue Mul_0 = Shl_0.getOperand(0); // Val
588 SDValue Mul_1 = Shl_0.getOperand(1); // Const
589 // RHS of mul is const.
590 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Mul_1)) {
591 int32_t ValConst = C->getSExtValue() << ShlConst;
592 if (isInt<9>(ValConst)) {
593 SDValue Val = CurDAG->getTargetConstant(ValConst, dl, MVT::i32);
594 SDNode *Result = CurDAG->getMachineNode(Hexagon::M2_mpysmi, dl,
595 MVT::i32, Mul_0, Val);
596 ReplaceNode(N, Result);
597 return;
598 }
599 }
600 return Default();
601 }
602
603 if (Shl_0.getOpcode() == ISD::SUB) {
604 SDValue Sub_0 = Shl_0.getOperand(0); // Const 0
605 SDValue Sub_1 = Shl_0.getOperand(1); // Val
606 if (ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Sub_0)) {
607 if (C1->getSExtValue() != 0 || Sub_1.getOpcode() != ISD::SHL)
608 return Default();
609 SDValue Shl2_0 = Sub_1.getOperand(0); // Val
610 SDValue Shl2_1 = Sub_1.getOperand(1); // Const
611 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(Shl2_1)) {
612 int32_t ValConst = 1 << (ShlConst + C2->getSExtValue());
613 if (isInt<9>(-ValConst)) {
614 SDValue Val = CurDAG->getTargetConstant(-ValConst, dl, MVT::i32);
615 SDNode *Result = CurDAG->getMachineNode(Hexagon::M2_mpysmi, dl,
616 MVT::i32, Shl2_0, Val);
617 ReplaceNode(N, Result);
618 return;
619 }
620 }
621 }
622 }
623
624 return Default();
625}
626
627//
628// Handling intrinsics for circular load and bitreverse load.
629//
634 return;
635 }
636
637 // Handle bit-reverse load intrinsics.
639 return;
640
642 return;
643
644 unsigned IntNo = N->getConstantOperandVal(1);
645 if (IntNo == Intrinsic::hexagon_V6_vgathermw ||
646 IntNo == Intrinsic::hexagon_V6_vgathermw_128B ||
647 IntNo == Intrinsic::hexagon_V6_vgathermh ||
648 IntNo == Intrinsic::hexagon_V6_vgathermh_128B ||
649 IntNo == Intrinsic::hexagon_V6_vgathermhw ||
650 IntNo == Intrinsic::hexagon_V6_vgathermhw_128B) {
652 return;
653 }
654 if (IntNo == Intrinsic::hexagon_V6_vgathermwq ||
655 IntNo == Intrinsic::hexagon_V6_vgathermwq_128B ||
656 IntNo == Intrinsic::hexagon_V6_vgathermhq ||
657 IntNo == Intrinsic::hexagon_V6_vgathermhq_128B ||
658 IntNo == Intrinsic::hexagon_V6_vgathermhwq ||
659 IntNo == Intrinsic::hexagon_V6_vgathermhwq_128B) {
661 return;
662 }
663
664 SelectCode(N);
665}
666
668 unsigned IID = N->getConstantOperandVal(0);
669 unsigned Bits;
670 switch (IID) {
671 case Intrinsic::hexagon_S2_vsplatrb:
672 Bits = 8;
673 break;
674 case Intrinsic::hexagon_S2_vsplatrh:
675 Bits = 16;
676 break;
677 case Intrinsic::hexagon_V6_vaddcarry:
678 case Intrinsic::hexagon_V6_vaddcarry_128B:
679 case Intrinsic::hexagon_V6_vsubcarry:
680 case Intrinsic::hexagon_V6_vsubcarry_128B:
682 return;
683 default:
684 SelectCode(N);
685 return;
686 }
687
688 SDValue V = N->getOperand(1);
689 SDValue U;
690 // Splat intrinsics.
691 if (keepsLowBits(V, Bits, U)) {
692 SDValue R = CurDAG->getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
693 N->getOperand(0), U);
694 ReplaceNode(N, R.getNode());
695 SelectCode(R.getNode());
696 return;
697 }
698 SelectCode(N);
699}
700
702 SDValue Inp = N->getOperand(0);
703 MVT ResTy = N->getValueType(0).getSimpleVT();
704 unsigned Idx = N->getConstantOperandVal(1);
705
706 [[maybe_unused]] MVT InpTy = Inp.getValueType().getSimpleVT();
707 [[maybe_unused]] unsigned ResLen = ResTy.getVectorNumElements();
709 assert(2 * ResLen == InpTy.getVectorNumElements());
710 assert(ResTy.getSizeInBits() == 32);
711 assert(Idx == 0 || Idx == ResLen);
712
713 unsigned SubReg = Idx == 0 ? Hexagon::isub_lo : Hexagon::isub_hi;
714 SDValue Ext = CurDAG->getTargetExtractSubreg(SubReg, SDLoc(N), ResTy, Inp);
715
716 ReplaceNode(N, Ext.getNode());
717}
718
719//
720// Map floating point constant values.
721//
723 SDLoc dl(N);
724 auto *CN = cast<ConstantFPSDNode>(N);
725 APInt A = CN->getValueAPF().bitcastToAPInt();
726 if (N->getValueType(0) == MVT::f32) {
727 SDValue V = CurDAG->getTargetConstant(A.getZExtValue(), dl, MVT::i32);
728 ReplaceNode(N, CurDAG->getMachineNode(Hexagon::A2_tfrsi, dl, MVT::f32, V));
729 return;
730 }
731 if (N->getValueType(0) == MVT::f64) {
732 SDValue V = CurDAG->getTargetConstant(A.getZExtValue(), dl, MVT::i64);
733 ReplaceNode(N, CurDAG->getMachineNode(Hexagon::CONST64, dl, MVT::f64, V));
734 return;
735 }
736
737 SelectCode(N);
738}
739
740//
741// Map boolean values.
742//
744 if (N->getValueType(0) == MVT::i1) {
745 assert(!(N->getAsZExtVal() >> 1));
746 unsigned Opc = (cast<ConstantSDNode>(N)->getSExtValue() != 0)
747 ? Hexagon::PS_true
748 : Hexagon::PS_false;
749 ReplaceNode(N, CurDAG->getMachineNode(Opc, SDLoc(N), MVT::i1));
750 return;
751 }
752
753 SelectCode(N);
754}
755
758 const HexagonFrameLowering *HFI = HST->getFrameLowering();
759 int FX = cast<FrameIndexSDNode>(N)->getIndex();
760 Align StkA = HFI->getStackAlign();
761 Align MaxA = MFI.getMaxAlign();
762 SDValue FI = CurDAG->getTargetFrameIndex(FX, MVT::i32);
763 SDLoc DL(N);
764 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
765 SDNode *R = nullptr;
766
767 // Use PS_fi when:
768 // - the object is fixed, or
769 // - there are no objects with higher-than-default alignment, or
770 // - there are no dynamically allocated objects.
771 // Otherwise, use PS_fia.
772 if (FX < 0 || MaxA <= StkA || !MFI.hasVarSizedObjects()) {
773 R = CurDAG->getMachineNode(Hexagon::PS_fi, DL, MVT::i32, FI, Zero);
774 } else {
775 auto &HMFI = *MF->getInfo<HexagonMachineFunctionInfo>();
776 Register AR = HMFI.getStackAlignBaseReg();
778 SDValue Ops[] = { CurDAG->getCopyFromReg(CH, DL, AR, MVT::i32), FI, Zero };
779 R = CurDAG->getMachineNode(Hexagon::PS_fia, DL, MVT::i32, Ops);
780 }
781
782 ReplaceNode(N, R);
783}
784
786 unsigned OpcCarry = N->getOpcode() == HexagonISD::ADDC ? Hexagon::A4_addp_c
787 : Hexagon::A4_subp_c;
788 SDNode *C = CurDAG->getMachineNode(OpcCarry, SDLoc(N), N->getVTList(),
789 { N->getOperand(0), N->getOperand(1),
790 N->getOperand(2) });
791 ReplaceNode(N, C);
792}
793
795 MVT ResTy = N->getValueType(0).getSimpleVT();
796 if (HST->isHVXVectorType(ResTy, true))
797 return SelectHvxVAlign(N);
798
799 const SDLoc &dl(N);
800 unsigned VecLen = ResTy.getSizeInBits();
801 if (VecLen == 32) {
802 SDValue Ops[] = {
803 CurDAG->getTargetConstant(Hexagon::DoubleRegsRegClassID, dl, MVT::i32),
804 N->getOperand(0),
805 CurDAG->getTargetConstant(Hexagon::isub_hi, dl, MVT::i32),
806 N->getOperand(1),
807 CurDAG->getTargetConstant(Hexagon::isub_lo, dl, MVT::i32)
808 };
809 SDNode *R = CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl,
810 MVT::i64, Ops);
811
812 // Shift right by "(Addr & 0x3) * 8" bytes.
813 SDNode *C;
814 SDValue M0 = CurDAG->getTargetConstant(0x18, dl, MVT::i32);
815 SDValue M1 = CurDAG->getTargetConstant(0x03, dl, MVT::i32);
816 if (HST->useCompound()) {
817 C = CurDAG->getMachineNode(Hexagon::S4_andi_asl_ri, dl, MVT::i32,
818 M0, N->getOperand(2), M1);
819 } else {
820 SDNode *T = CurDAG->getMachineNode(Hexagon::S2_asl_i_r, dl, MVT::i32,
821 N->getOperand(2), M1);
822 C = CurDAG->getMachineNode(Hexagon::A2_andir, dl, MVT::i32,
823 SDValue(T, 0), M0);
824 }
825 SDNode *S = CurDAG->getMachineNode(Hexagon::S2_lsr_r_p, dl, MVT::i64,
826 SDValue(R, 0), SDValue(C, 0));
827 SDValue E = CurDAG->getTargetExtractSubreg(Hexagon::isub_lo, dl, ResTy,
828 SDValue(S, 0));
829 ReplaceNode(N, E.getNode());
830 } else {
831 assert(VecLen == 64);
832 SDNode *Pu = CurDAG->getMachineNode(Hexagon::C2_tfrrp, dl, MVT::v8i1,
833 N->getOperand(2));
834 SDNode *VA = CurDAG->getMachineNode(Hexagon::S2_valignrb, dl, ResTy,
835 N->getOperand(0), N->getOperand(1),
836 SDValue(Pu,0));
837 ReplaceNode(N, VA);
838 }
839}
840
842 const SDLoc &dl(N);
843 SDValue A = N->getOperand(1);
844 int Mask = -cast<ConstantSDNode>(A.getNode())->getSExtValue();
845 assert(isPowerOf2_32(-Mask));
846
847 SDValue M = CurDAG->getTargetConstant(Mask, dl, MVT::i32);
848 SDNode *AA = CurDAG->getMachineNode(Hexagon::A2_andir, dl, MVT::i32,
849 N->getOperand(0), M);
850 ReplaceNode(N, AA);
851}
852
853// Handle these nodes here to avoid having to write patterns for all
854// combinations of input/output types. In all cases, the resulting
855// instruction is the same.
857 SDValue Op = N->getOperand(0);
858 MVT OpTy = Op.getValueType().getSimpleVT();
859 SDNode *T = CurDAG->MorphNodeTo(N, N->getOpcode(),
860 CurDAG->getVTList(OpTy), {Op});
861 ReplaceNode(T, Op.getNode());
862}
863
865 MVT ResTy = N->getValueType(0).getSimpleVT();
866 SDNode *T = CurDAG->getMachineNode(Hexagon::C2_mask, SDLoc(N), ResTy,
867 N->getOperand(0));
868 ReplaceNode(N, T);
869}
870
872 const SDLoc &dl(N);
873 MVT ResTy = N->getValueType(0).getSimpleVT();
874 SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32);
875 SDNode *T = CurDAG->getMachineNode(Hexagon::A4_vcmpbgtui, dl, ResTy,
876 N->getOperand(0), Zero);
877 ReplaceNode(N, T);
878}
879
881 const SDLoc &dl(N);
882 MVT ResTy = N->getValueType(0).getSimpleVT();
883 // The argument to V2Q should be a single vector.
884 MVT OpTy = N->getOperand(0).getValueType().getSimpleVT(); (void)OpTy;
885 assert(HST->getVectorLength() * 8 == OpTy.getSizeInBits());
886
887 SDValue C = CurDAG->getTargetConstant(-1, dl, MVT::i32);
888 SDNode *R = CurDAG->getMachineNode(Hexagon::A2_tfrsi, dl, MVT::i32, C);
889 SDNode *T = CurDAG->getMachineNode(Hexagon::V6_vandvrt, dl, ResTy,
890 N->getOperand(0), SDValue(R,0));
891 ReplaceNode(N, T);
892}
893
895 const SDLoc &dl(N);
896 MVT ResTy = N->getValueType(0).getSimpleVT();
897 // The result of V2Q should be a single vector.
898 assert(HST->getVectorLength() * 8 == ResTy.getSizeInBits());
899
900 SDValue C = CurDAG->getTargetConstant(-1, dl, MVT::i32);
901 SDNode *R = CurDAG->getMachineNode(Hexagon::A2_tfrsi, dl, MVT::i32, C);
902 SDNode *T = CurDAG->getMachineNode(Hexagon::V6_vandqrt, dl, ResTy,
903 N->getOperand(0), SDValue(R,0));
904 ReplaceNode(N, T);
905}
906
908 const SDLoc &dl(N);
909 ArrayRef<EVT> ResultType(N->value_begin(), N->value_end());
911 Ops = {N->getOperand(0), N->getOperand(1)};
912 SDVTList VTs;
913 VTs = CurDAG->getVTList(MVT::f32, MVT::f32);
914 SDNode *ResScale = CurDAG->getMachineNode(Hexagon::F2_sfrecipa, dl, VTs, Ops);
915 SDNode *D = CurDAG->getMachineNode(Hexagon::F2_sffixupd, dl, MVT::f32, Ops);
916
917 SDValue C = CurDAG->getTargetConstant(0x3f800000, dl, MVT::i32);
918 SDNode *constNode =
919 CurDAG->getMachineNode(Hexagon::A2_tfrsi, dl, MVT::f32, C);
920
921 SDNode *n = CurDAG->getMachineNode(Hexagon::F2_sffixupn, dl, MVT::f32, Ops);
922 SDNode *Err = CurDAG->getMachineNode(Hexagon::F2_sffms_lib, dl, MVT::f32,
923 SDValue(constNode, 0), SDValue(D, 0),
924 SDValue(ResScale, 0));
925 SDNode *NewRec = CurDAG->getMachineNode(Hexagon::F2_sffma_lib, dl, MVT::f32,
926 SDValue(ResScale, 0), SDValue(Err, 0),
927 SDValue(ResScale, 0));
928 SDNode *newErr = CurDAG->getMachineNode(Hexagon::F2_sffms_lib, dl, MVT::f32,
929 SDValue(constNode, 0), SDValue(D, 0),
930 SDValue(NewRec, 0));
932 Hexagon::A2_andir, dl, MVT::f32, SDValue(n, 0),
933 CurDAG->getTargetConstant(0x80000000, dl, MVT::i32));
934 SDNode *NewQ =
935 CurDAG->getMachineNode(Hexagon::F2_sffma_lib, dl, MVT::f32, SDValue(q, 0),
936 SDValue(n, 0), SDValue(NewRec, 0));
937 SDNode *NNewRec = CurDAG->getMachineNode(
938 Hexagon::F2_sffma_lib, dl, MVT::f32, SDValue(NewRec, 0),
939 SDValue(newErr, 0), SDValue(NewRec, 0));
940 SDNode *qErr =
941 CurDAG->getMachineNode(Hexagon::F2_sffms_lib, dl, MVT::f32, SDValue(n, 0),
942 SDValue(D, 0), SDValue(NewQ, 0));
943 SDNode *NNewQ = CurDAG->getMachineNode(Hexagon::F2_sffma_lib, dl, MVT::f32,
944 SDValue(NewQ, 0), SDValue(qErr, 0),
945 SDValue(NNewRec, 0));
946
947 SDNode *NqErr =
948 CurDAG->getMachineNode(Hexagon::F2_sffms_lib, dl, MVT::f32, SDValue(n, 0),
949 SDValue(NNewQ, 0), SDValue(D, 0));
950 std::array<SDValue, 4> temp1 = {SDValue(NNewQ, 0), SDValue(NqErr, 0),
951 SDValue(NNewRec, 0), SDValue(ResScale, 1)};
952 ArrayRef<SDValue> OpValue1(temp1);
953 SDNode *FinalNewQ =
954 CurDAG->getMachineNode(Hexagon::F2_sffma_sc, dl, MVT::f32, OpValue1);
955 ReplaceNode(N, FinalNewQ);
956}
957
959 const SDLoc &dl(N);
960 ArrayRef<EVT> ResultType(N->value_begin(), N->value_end());
962 Ops = {N->getOperand(0), N->getOperand(1)};
963 SDVTList VTs;
964 VTs = CurDAG->getVTList(MVT::f32, MVT::f32);
965 SDNode *ResScale = CurDAG->getMachineNode(Hexagon::F2_sfrecipa, dl, VTs, Ops);
966 SDNode *D = CurDAG->getMachineNode(Hexagon::F2_sffixupd, dl, MVT::f32, Ops);
967
968 SDValue C = CurDAG->getTargetConstant(0x3f800000, dl, MVT::i32);
969 SDNode *constNode =
970 CurDAG->getMachineNode(Hexagon::A2_tfrsi, dl, MVT::f32, C);
971
972 SDNode *n = CurDAG->getMachineNode(Hexagon::F2_sffixupn, dl, MVT::f32, Ops);
973 SDNode *Err = CurDAG->getMachineNode(Hexagon::F2_sffms_lib, dl, MVT::f32,
974 SDValue(constNode, 0), SDValue(D, 0),
975 SDValue(ResScale, 0));
976 SDNode *NewRec = CurDAG->getMachineNode(Hexagon::F2_sffma_lib, dl, MVT::f32,
977 SDValue(ResScale, 0), SDValue(Err, 0),
978 SDValue(ResScale, 0));
979 SDNode *newErr = CurDAG->getMachineNode(Hexagon::F2_sffms_lib, dl, MVT::f32,
980 SDValue(constNode, 0), SDValue(D, 0),
981 SDValue(NewRec, 0));
982
983 SDNode *NNewRec = CurDAG->getMachineNode(
984 Hexagon::F2_sffma_lib, dl, MVT::f32, SDValue(NewRec, 0),
985 SDValue(newErr, 0), SDValue(NewRec, 0));
986 SDNode *FinalNewQ = CurDAG->getMachineNode(
987 Hexagon::F2_sfmpy, dl, MVT::f32, SDValue(NNewRec, 0), SDValue(n, 0));
988 ReplaceNode(N, FinalNewQ);
989}
990
992 if (N->getFlags().hasAllowReassociation())
993 FastFDiv(N);
994 else
995 FDiv(N);
996 return;
997}
998
1000 if (N->isMachineOpcode())
1001 return N->setNodeId(-1); // Already selected.
1002
1003 auto isHvxOp = [this](SDNode *N) {
1004 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
1005 if (HST->isHVXVectorType(N->getValueType(i), true))
1006 return true;
1007 }
1008 for (SDValue I : N->ops()) {
1009 if (HST->isHVXVectorType(I.getValueType(), true))
1010 return true;
1011 }
1012 return false;
1013 };
1014
1015 if (HST->useHVXOps() && isHvxOp(N)) {
1016 switch (N->getOpcode()) {
1017 case ISD::EXTRACT_SUBVECTOR: return SelectHvxExtractSubvector(N);
1018 case ISD::VECTOR_SHUFFLE: return SelectHvxShuffle(N);
1019
1020 case HexagonISD::VROR: return SelectHvxRor(N);
1021 }
1022 }
1023
1024 switch (N->getOpcode()) {
1025 case ISD::Constant: return SelectConstant(N);
1026 case ISD::ConstantFP: return SelectConstantFP(N);
1027 case ISD::FrameIndex: return SelectFrameIndex(N);
1028 case ISD::SHL: return SelectSHL(N);
1029 case ISD::LOAD: return SelectLoad(N);
1030 case ISD::STORE: return SelectStore(N);
1034
1035 case HexagonISD::ADDC:
1036 case HexagonISD::SUBC: return SelectAddSubCarry(N);
1037 case HexagonISD::VALIGN: return SelectVAlign(N);
1040 case HexagonISD::P2D: return SelectP2D(N);
1041 case HexagonISD::D2P: return SelectD2P(N);
1042 case HexagonISD::Q2V: return SelectQ2V(N);
1043 case HexagonISD::V2Q: return SelectV2Q(N);
1044 case ISD::FDIV:
1045 return SelectFDiv(N);
1046 }
1047
1048 SelectCode(N);
1049}
1050
1052 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
1053 std::vector<SDValue> &OutOps) {
1054 SDValue Inp = Op, Res;
1055
1056 switch (ConstraintID) {
1057 default:
1058 return true;
1059 case InlineAsm::ConstraintCode::o: // Offsetable.
1060 case InlineAsm::ConstraintCode::v: // Not offsetable.
1061 case InlineAsm::ConstraintCode::m: // Memory.
1062 if (SelectAddrFI(Inp, Res))
1063 OutOps.push_back(Res);
1064 else
1065 OutOps.push_back(Inp);
1066 break;
1067 }
1068
1069 OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
1070 return false;
1071}
1072
1073static bool isMemOPCandidate(SDNode *I, SDNode *U) {
1074 // I is an operand of U. Check if U is an arithmetic (binary) operation
1075 // usable in a memop, where the other operand is a loaded value, and the
1076 // result of U is stored in the same location.
1077
1078 if (!U->hasOneUse())
1079 return false;
1080 unsigned Opc = U->getOpcode();
1081 switch (Opc) {
1082 case ISD::ADD:
1083 case ISD::SUB:
1084 case ISD::AND:
1085 case ISD::OR:
1086 break;
1087 default:
1088 return false;
1089 }
1090
1091 SDValue S0 = U->getOperand(0);
1092 SDValue S1 = U->getOperand(1);
1093 SDValue SY = (S0.getNode() == I) ? S1 : S0;
1094
1095 SDNode *UUse = *U->use_begin();
1096 if (UUse->getNumValues() != 1)
1097 return false;
1098
1099 // Check if one of the inputs to U is a load instruction and the output
1100 // is used by a store instruction. If so and they also have the same
1101 // base pointer, then don't preoprocess this node sequence as it
1102 // can be matched to a memop.
1103 SDNode *SYNode = SY.getNode();
1104 if (UUse->getOpcode() == ISD::STORE && SYNode->getOpcode() == ISD::LOAD) {
1105 SDValue LDBasePtr = cast<MemSDNode>(SYNode)->getBasePtr();
1106 SDValue STBasePtr = cast<MemSDNode>(UUse)->getBasePtr();
1107 if (LDBasePtr == STBasePtr)
1108 return true;
1109 }
1110 return false;
1111}
1112
1113
1114// Transform: (or (select c x 0) z) -> (select c (or x z) z)
1115// (or (select c 0 y) z) -> (select c z (or y z))
1116void HexagonDAGToDAGISel::ppSimplifyOrSelect0(std::vector<SDNode*> &&Nodes) {
1117 SelectionDAG &DAG = *CurDAG;
1118
1119 for (auto *I : Nodes) {
1120 if (I->getOpcode() != ISD::OR)
1121 continue;
1122
1123 auto IsSelect0 = [](const SDValue &Op) -> bool {
1124 if (Op.getOpcode() != ISD::SELECT)
1125 return false;
1126 return isNullConstant(Op.getOperand(1)) ||
1127 isNullConstant(Op.getOperand(2));
1128 };
1129
1130 SDValue N0 = I->getOperand(0), N1 = I->getOperand(1);
1131 EVT VT = I->getValueType(0);
1132 bool SelN0 = IsSelect0(N0);
1133 SDValue SOp = SelN0 ? N0 : N1;
1134 SDValue VOp = SelN0 ? N1 : N0;
1135
1136 if (SOp.getOpcode() == ISD::SELECT && SOp.getNode()->hasOneUse()) {
1137 SDValue SC = SOp.getOperand(0);
1138 SDValue SX = SOp.getOperand(1);
1139 SDValue SY = SOp.getOperand(2);
1140 SDLoc DLS = SOp;
1141 if (isNullConstant(SY)) {
1142 SDValue NewOr = DAG.getNode(ISD::OR, DLS, VT, SX, VOp);
1143 SDValue NewSel = DAG.getNode(ISD::SELECT, DLS, VT, SC, NewOr, VOp);
1144 DAG.ReplaceAllUsesWith(I, NewSel.getNode());
1145 } else if (isNullConstant(SX)) {
1146 SDValue NewOr = DAG.getNode(ISD::OR, DLS, VT, SY, VOp);
1147 SDValue NewSel = DAG.getNode(ISD::SELECT, DLS, VT, SC, VOp, NewOr);
1148 DAG.ReplaceAllUsesWith(I, NewSel.getNode());
1149 }
1150 }
1151 }
1152}
1153
1154// Transform: (store ch val (add x (add (shl y c) e)))
1155// to: (store ch val (add x (shl (add y d) c))),
1156// where e = (shl d c) for some integer d.
1157// The purpose of this is to enable generation of loads/stores with
1158// shifted addressing mode, i.e. mem(x+y<<#c). For that, the shift
1159// value c must be 0, 1 or 2.
1160void HexagonDAGToDAGISel::ppAddrReorderAddShl(std::vector<SDNode*> &&Nodes) {
1161 SelectionDAG &DAG = *CurDAG;
1162
1163 for (auto *I : Nodes) {
1164 if (I->getOpcode() != ISD::STORE)
1165 continue;
1166
1167 // I matched: (store ch val Off)
1168 SDValue Off = I->getOperand(2);
1169 // Off needs to match: (add x (add (shl y c) (shl d c))))
1170 if (Off.getOpcode() != ISD::ADD)
1171 continue;
1172 // Off matched: (add x T0)
1173 SDValue T0 = Off.getOperand(1);
1174 // T0 needs to match: (add T1 T2):
1175 if (T0.getOpcode() != ISD::ADD)
1176 continue;
1177 // T0 matched: (add T1 T2)
1178 SDValue T1 = T0.getOperand(0);
1179 SDValue T2 = T0.getOperand(1);
1180 // T1 needs to match: (shl y c)
1181 if (T1.getOpcode() != ISD::SHL)
1182 continue;
1183 SDValue C = T1.getOperand(1);
1184 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(C.getNode());
1185 if (CN == nullptr)
1186 continue;
1187 unsigned CV = CN->getZExtValue();
1188 if (CV > 2)
1189 continue;
1190 // T2 needs to match e, where e = (shl d c) for some d.
1191 ConstantSDNode *EN = dyn_cast<ConstantSDNode>(T2.getNode());
1192 if (EN == nullptr)
1193 continue;
1194 unsigned EV = EN->getZExtValue();
1195 if (EV % (1 << CV) != 0)
1196 continue;
1197 unsigned DV = EV / (1 << CV);
1198
1199 // Replace T0 with: (shl (add y d) c)
1200 SDLoc DL = SDLoc(I);
1201 EVT VT = T0.getValueType();
1202 SDValue D = DAG.getConstant(DV, DL, VT);
1203 // NewAdd = (add y d)
1204 SDValue NewAdd = DAG.getNode(ISD::ADD, DL, VT, T1.getOperand(0), D);
1205 // NewShl = (shl NewAdd c)
1206 SDValue NewShl = DAG.getNode(ISD::SHL, DL, VT, NewAdd, C);
1207 ReplaceNode(T0.getNode(), NewShl.getNode());
1208 }
1209}
1210
1211// Transform: (load ch (add x (and (srl y c) Mask)))
1212// to: (load ch (add x (shl (srl y d) d-c)))
1213// where
1214// Mask = 00..0 111..1 0.0
1215// | | +-- d-c 0s, and d-c is 0, 1 or 2.
1216// | +-------- 1s
1217// +-------------- at most c 0s
1218// Motivating example:
1219// DAG combiner optimizes (add x (shl (srl y 5) 2))
1220// to (add x (and (srl y 3) 1FFFFFFC))
1221// which results in a constant-extended and(##...,lsr). This transformation
1222// undoes this simplification for cases where the shl can be folded into
1223// an addressing mode.
1224void HexagonDAGToDAGISel::ppAddrRewriteAndSrl(std::vector<SDNode*> &&Nodes) {
1225 SelectionDAG &DAG = *CurDAG;
1226
1227 for (SDNode *N : Nodes) {
1228 unsigned Opc = N->getOpcode();
1229 if (Opc != ISD::LOAD && Opc != ISD::STORE)
1230 continue;
1231 SDValue Addr = Opc == ISD::LOAD ? N->getOperand(1) : N->getOperand(2);
1232 // Addr must match: (add x T0)
1233 if (Addr.getOpcode() != ISD::ADD)
1234 continue;
1235 SDValue T0 = Addr.getOperand(1);
1236 // T0 must match: (and T1 Mask)
1237 if (T0.getOpcode() != ISD::AND)
1238 continue;
1239
1240 // We have an AND.
1241 //
1242 // Check the first operand. It must be: (srl y c).
1243 SDValue S = T0.getOperand(0);
1244 if (S.getOpcode() != ISD::SRL)
1245 continue;
1246 ConstantSDNode *SN = dyn_cast<ConstantSDNode>(S.getOperand(1).getNode());
1247 if (SN == nullptr)
1248 continue;
1249 if (SN->getAPIntValue().getBitWidth() != 32)
1250 continue;
1251 uint32_t CV = SN->getZExtValue();
1252
1253 // Check the second operand: the supposed mask.
1254 ConstantSDNode *MN = dyn_cast<ConstantSDNode>(T0.getOperand(1).getNode());
1255 if (MN == nullptr)
1256 continue;
1257 if (MN->getAPIntValue().getBitWidth() != 32)
1258 continue;
1259 uint32_t Mask = MN->getZExtValue();
1260 // Examine the mask.
1261 uint32_t TZ = llvm::countr_zero(Mask);
1262 uint32_t M1 = llvm::countr_one(Mask >> TZ);
1263 uint32_t LZ = llvm::countl_zero(Mask);
1264 // Trailing zeros + middle ones + leading zeros must equal the width.
1265 if (TZ + M1 + LZ != 32)
1266 continue;
1267 // The number of trailing zeros will be encoded in the addressing mode.
1268 if (TZ > 2)
1269 continue;
1270 // The number of leading zeros must be at most c.
1271 if (LZ > CV)
1272 continue;
1273
1274 // All looks good.
1275 SDValue Y = S.getOperand(0);
1276 EVT VT = Addr.getValueType();
1277 SDLoc dl(S);
1278 // TZ = D-C, so D = TZ+C.
1279 SDValue D = DAG.getConstant(TZ+CV, dl, VT);
1280 SDValue DC = DAG.getConstant(TZ, dl, VT);
1281 SDValue NewSrl = DAG.getNode(ISD::SRL, dl, VT, Y, D);
1282 SDValue NewShl = DAG.getNode(ISD::SHL, dl, VT, NewSrl, DC);
1283 ReplaceNode(T0.getNode(), NewShl.getNode());
1284 }
1285}
1286
1287// Transform: (op ... (zext i1 c) ...) -> (select c (op ... 0 ...)
1288// (op ... 1 ...))
1289void HexagonDAGToDAGISel::ppHoistZextI1(std::vector<SDNode*> &&Nodes) {
1290 SelectionDAG &DAG = *CurDAG;
1291
1292 for (SDNode *N : Nodes) {
1293 unsigned Opc = N->getOpcode();
1294 if (Opc != ISD::ZERO_EXTEND)
1295 continue;
1296 SDValue OpI1 = N->getOperand(0);
1297 EVT OpVT = OpI1.getValueType();
1298 if (!OpVT.isSimple() || OpVT.getSimpleVT() != MVT::i1)
1299 continue;
1300 for (auto I = N->use_begin(), E = N->use_end(); I != E; ++I) {
1301 SDNode *U = *I;
1302 if (U->getNumValues() != 1)
1303 continue;
1304 EVT UVT = U->getValueType(0);
1305 if (!UVT.isSimple() || !UVT.isInteger() || UVT.getSimpleVT() == MVT::i1)
1306 continue;
1307 // Do not generate select for all i1 vector type.
1308 if (UVT.isVector() && UVT.getVectorElementType() == MVT::i1)
1309 continue;
1310 if (isMemOPCandidate(N, U))
1311 continue;
1312
1313 // Potentially simplifiable operation.
1314 unsigned I1N = I.getOperandNo();
1315 SmallVector<SDValue,2> Ops(U->getNumOperands());
1316 for (unsigned i = 0, n = U->getNumOperands(); i != n; ++i)
1317 Ops[i] = U->getOperand(i);
1318 EVT BVT = Ops[I1N].getValueType();
1319
1320 const SDLoc &dl(U);
1321 SDValue C0 = DAG.getConstant(0, dl, BVT);
1322 SDValue C1 = DAG.getConstant(1, dl, BVT);
1323 SDValue If0, If1;
1324
1325 if (isa<MachineSDNode>(U)) {
1326 unsigned UseOpc = U->getMachineOpcode();
1327 Ops[I1N] = C0;
1328 If0 = SDValue(DAG.getMachineNode(UseOpc, dl, UVT, Ops), 0);
1329 Ops[I1N] = C1;
1330 If1 = SDValue(DAG.getMachineNode(UseOpc, dl, UVT, Ops), 0);
1331 } else {
1332 unsigned UseOpc = U->getOpcode();
1333 Ops[I1N] = C0;
1334 If0 = DAG.getNode(UseOpc, dl, UVT, Ops);
1335 Ops[I1N] = C1;
1336 If1 = DAG.getNode(UseOpc, dl, UVT, Ops);
1337 }
1338 // We're generating a SELECT way after legalization, so keep the types
1339 // simple.
1340 unsigned UW = UVT.getSizeInBits();
1341 EVT SVT = (UW == 32 || UW == 64) ? MVT::getIntegerVT(UW) : UVT;
1342 SDValue Sel = DAG.getNode(ISD::SELECT, dl, SVT, OpI1,
1343 DAG.getBitcast(SVT, If1),
1344 DAG.getBitcast(SVT, If0));
1345 SDValue Ret = DAG.getBitcast(UVT, Sel);
1346 DAG.ReplaceAllUsesWith(U, Ret.getNode());
1347 }
1348 }
1349}
1350
1352 // Repack all nodes before calling each preprocessing function,
1353 // because each of them can modify the set of nodes.
1354 auto getNodes = [this]() -> std::vector<SDNode *> {
1355 std::vector<SDNode *> T;
1356 T.reserve(CurDAG->allnodes_size());
1357 for (SDNode &N : CurDAG->allnodes())
1358 T.push_back(&N);
1359 return T;
1360 };
1361
1362 if (HST->useHVXOps())
1363 PreprocessHvxISelDAG();
1364
1365 // Transform: (or (select c x 0) z) -> (select c (or x z) z)
1366 // (or (select c 0 y) z) -> (select c z (or y z))
1367 ppSimplifyOrSelect0(getNodes());
1368
1369 // Transform: (store ch val (add x (add (shl y c) e)))
1370 // to: (store ch val (add x (shl (add y d) c))),
1371 // where e = (shl d c) for some integer d.
1372 // The purpose of this is to enable generation of loads/stores with
1373 // shifted addressing mode, i.e. mem(x+y<<#c). For that, the shift
1374 // value c must be 0, 1 or 2.
1375 ppAddrReorderAddShl(getNodes());
1376
1377 // Transform: (load ch (add x (and (srl y c) Mask)))
1378 // to: (load ch (add x (shl (srl y d) d-c)))
1379 // where
1380 // Mask = 00..0 111..1 0.0
1381 // | | +-- d-c 0s, and d-c is 0, 1 or 2.
1382 // | +-------- 1s
1383 // +-------------- at most c 0s
1384 // Motivating example:
1385 // DAG combiner optimizes (add x (shl (srl y 5) 2))
1386 // to (add x (and (srl y 3) 1FFFFFFC))
1387 // which results in a constant-extended and(##...,lsr). This transformation
1388 // undoes this simplification for cases where the shl can be folded into
1389 // an addressing mode.
1390 ppAddrRewriteAndSrl(getNodes());
1391
1392 // Transform: (op ... (zext i1 c) ...) -> (select c (op ... 0 ...)
1393 // (op ... 1 ...))
1394 ppHoistZextI1(getNodes());
1395
1396 DEBUG_WITH_TYPE("isel", {
1397 dbgs() << "Preprocessed (Hexagon) selection DAG:";
1398 CurDAG->dump();
1399 });
1400
1402 rebalanceAddressTrees();
1403
1404 DEBUG_WITH_TYPE("isel", {
1405 dbgs() << "Address tree balanced selection DAG:";
1406 CurDAG->dump();
1407 });
1408 }
1409}
1410
1412 auto &HST = MF->getSubtarget<HexagonSubtarget>();
1413 auto &HFI = *HST.getFrameLowering();
1414 if (!HFI.needsAligna(*MF))
1415 return;
1416
1418 MachineBasicBlock *EntryBB = &MF->front();
1419 Align EntryMaxA = MFI.getMaxAlign();
1420
1421 // Reserve the first non-volatile register.
1422 Register AP = 0;
1423 auto &HRI = *HST.getRegisterInfo();
1425 for (const MCPhysReg *R = HRI.getCalleeSavedRegs(MF); *R; ++R) {
1426 if (Reserved[*R])
1427 continue;
1428 AP = *R;
1429 break;
1430 }
1431 assert(AP.isValid() && "Couldn't reserve stack align register");
1432 BuildMI(EntryBB, DebugLoc(), HII->get(Hexagon::PS_aligna), AP)
1433 .addImm(EntryMaxA.value());
1434 MF->getInfo<HexagonMachineFunctionInfo>()->setStackAlignBaseReg(AP);
1435}
1436
1437void HexagonDAGToDAGISel::updateAligna() {
1438 auto &HFI = *MF->getSubtarget<HexagonSubtarget>().getFrameLowering();
1439 if (!HFI.needsAligna(*MF))
1440 return;
1441 auto *AlignaI = const_cast<MachineInstr*>(HFI.getAlignaInstr(*MF));
1442 assert(AlignaI != nullptr);
1443 unsigned MaxA = MF->getFrameInfo().getMaxAlign().value();
1444 if (AlignaI->getOperand(1).getImm() < MaxA)
1445 AlignaI->getOperand(1).setImm(MaxA);
1446}
1447
1448// Match a frame index that can be used in an addressing mode.
1450 if (N.getOpcode() != ISD::FrameIndex)
1451 return false;
1452 auto &HFI = *HST->getFrameLowering();
1454 int FX = cast<FrameIndexSDNode>(N)->getIndex();
1455 if (!MFI.isFixedObjectIndex(FX) && HFI.needsAligna(*MF))
1456 return false;
1457 R = CurDAG->getTargetFrameIndex(FX, MVT::i32);
1458 return true;
1459}
1460
1462 return SelectGlobalAddress(N, R, false, Align(1));
1463}
1464
1466 return SelectGlobalAddress(N, R, true, Align(1));
1467}
1468
1470 return SelectAnyImmediate(N, R, Align(1));
1471}
1472
1474 return SelectAnyImmediate(N, R, Align(1));
1475}
1477 return SelectAnyImmediate(N, R, Align(2));
1478}
1480 return SelectAnyImmediate(N, R, Align(4));
1481}
1483 return SelectAnyImmediate(N, R, Align(8));
1484}
1485
1487 EVT T = N.getValueType();
1488 if (!T.isInteger() || T.getSizeInBits() != 32 || !isa<ConstantSDNode>(N))
1489 return false;
1490 int32_t V = cast<const ConstantSDNode>(N)->getZExtValue();
1491 R = CurDAG->getTargetConstant(V, SDLoc(N), N.getValueType());
1492 return true;
1493}
1494
1496 Align Alignment) {
1497 switch (N.getOpcode()) {
1498 case ISD::Constant: {
1499 if (N.getValueType() != MVT::i32)
1500 return false;
1501 int32_t V = cast<const ConstantSDNode>(N)->getZExtValue();
1502 if (!isAligned(Alignment, V))
1503 return false;
1504 R = CurDAG->getTargetConstant(V, SDLoc(N), N.getValueType());
1505 return true;
1506 }
1507 case HexagonISD::JT:
1508 case HexagonISD::CP:
1509 // These are assumed to always be aligned at least 8-byte boundary.
1510 if (Alignment > Align(8))
1511 return false;
1512 R = N.getOperand(0);
1513 return true;
1515 // Symbols may be aligned at any boundary.
1516 if (Alignment > Align(1))
1517 return false;
1518 R = N;
1519 return true;
1520 case ISD::BlockAddress:
1521 // Block address is always aligned at least 4-byte boundary.
1522 if (Alignment > Align(4) ||
1523 !isAligned(Alignment, cast<BlockAddressSDNode>(N)->getOffset()))
1524 return false;
1525 R = N;
1526 return true;
1527 }
1528
1529 if (SelectGlobalAddress(N, R, false, Alignment) ||
1530 SelectGlobalAddress(N, R, true, Alignment))
1531 return true;
1532
1533 return false;
1534}
1535
1537 bool UseGP, Align Alignment) {
1538 switch (N.getOpcode()) {
1539 case ISD::ADD: {
1540 SDValue N0 = N.getOperand(0);
1541 SDValue N1 = N.getOperand(1);
1542 unsigned GAOpc = N0.getOpcode();
1543 if (UseGP && GAOpc != HexagonISD::CONST32_GP)
1544 return false;
1545 if (!UseGP && GAOpc != HexagonISD::CONST32)
1546 return false;
1547 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N1)) {
1548 if (!isAligned(Alignment, Const->getZExtValue()))
1549 return false;
1550 SDValue Addr = N0.getOperand(0);
1551 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Addr)) {
1552 if (GA->getOpcode() == ISD::TargetGlobalAddress) {
1553 uint64_t NewOff = GA->getOffset() + (uint64_t)Const->getSExtValue();
1554 R = CurDAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(Const),
1555 N.getValueType(), NewOff);
1556 return true;
1557 }
1558 }
1559 }
1560 break;
1561 }
1562 case HexagonISD::CP:
1563 case HexagonISD::JT:
1565 // The operand(0) of CONST32 is TargetGlobalAddress, which is what we
1566 // want in the instruction.
1567 if (!UseGP)
1568 R = N.getOperand(0);
1569 return !UseGP;
1571 if (UseGP)
1572 R = N.getOperand(0);
1573 return UseGP;
1574 default:
1575 return false;
1576 }
1577
1578 return false;
1579}
1580
1582 // This (complex pattern) function is meant to detect a sign-extension
1583 // i32->i64 on a per-operand basis. This would allow writing single
1584 // patterns that would cover a number of combinations of different ways
1585 // a sign-extensions could be written. For example:
1586 // (mul (DetectUseSxtw x) (DetectUseSxtw y)) -> (M2_dpmpyss_s0 x y)
1587 // could match either one of these:
1588 // (mul (sext x) (sext_inreg y))
1589 // (mul (sext-load *p) (sext_inreg y))
1590 // (mul (sext_inreg x) (sext y))
1591 // etc.
1592 //
1593 // The returned value will have type i64 and its low word will
1594 // contain the value being extended. The high bits are not specified.
1595 // The returned type is i64 because the original type of N was i64,
1596 // but the users of this function should only use the low-word of the
1597 // result, e.g.
1598 // (mul sxtw:x, sxtw:y) -> (M2_dpmpyss_s0 (LoReg sxtw:x), (LoReg sxtw:y))
1599
1600 if (N.getValueType() != MVT::i64)
1601 return false;
1602 unsigned Opc = N.getOpcode();
1603 switch (Opc) {
1604 case ISD::SIGN_EXTEND:
1606 // sext_inreg has the source type as a separate operand.
1607 EVT T = Opc == ISD::SIGN_EXTEND
1608 ? N.getOperand(0).getValueType()
1609 : cast<VTSDNode>(N.getOperand(1))->getVT();
1610 unsigned SW = T.getSizeInBits();
1611 if (SW == 32)
1612 R = N.getOperand(0);
1613 else if (SW < 32)
1614 R = N;
1615 else
1616 return false;
1617 break;
1618 }
1619 case ISD::LOAD: {
1620 LoadSDNode *L = cast<LoadSDNode>(N);
1621 if (L->getExtensionType() != ISD::SEXTLOAD)
1622 return false;
1623 // All extending loads extend to i32, so even if the value in
1624 // memory is shorter than 32 bits, it will be i32 after the load.
1625 if (L->getMemoryVT().getSizeInBits() > 32)
1626 return false;
1627 R = N;
1628 break;
1629 }
1630 case ISD::SRA: {
1631 auto *S = dyn_cast<ConstantSDNode>(N.getOperand(1));
1632 if (!S || S->getZExtValue() != 32)
1633 return false;
1634 R = N;
1635 break;
1636 }
1637 default:
1638 return false;
1639 }
1640 EVT RT = R.getValueType();
1641 if (RT == MVT::i64)
1642 return true;
1643 assert(RT == MVT::i32);
1644 // This is only to produce a value of type i64. Do not rely on the
1645 // high bits produced by this.
1646 const SDLoc &dl(N);
1647 SDValue Ops[] = {
1648 CurDAG->getTargetConstant(Hexagon::DoubleRegsRegClassID, dl, MVT::i32),
1649 R, CurDAG->getTargetConstant(Hexagon::isub_hi, dl, MVT::i32),
1650 R, CurDAG->getTargetConstant(Hexagon::isub_lo, dl, MVT::i32)
1651 };
1652 SDNode *T = CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl,
1653 MVT::i64, Ops);
1654 R = SDValue(T, 0);
1655 return true;
1656}
1657
1658bool HexagonDAGToDAGISel::keepsLowBits(const SDValue &Val, unsigned NumBits,
1659 SDValue &Src) {
1660 unsigned Opc = Val.getOpcode();
1661 switch (Opc) {
1662 case ISD::SIGN_EXTEND:
1663 case ISD::ZERO_EXTEND:
1664 case ISD::ANY_EXTEND: {
1665 const SDValue &Op0 = Val.getOperand(0);
1666 EVT T = Op0.getValueType();
1667 if (T.isInteger() && T.getSizeInBits() == NumBits) {
1668 Src = Op0;
1669 return true;
1670 }
1671 break;
1672 }
1674 case ISD::AssertSext:
1675 case ISD::AssertZext:
1676 if (Val.getOperand(0).getValueType().isInteger()) {
1677 VTSDNode *T = cast<VTSDNode>(Val.getOperand(1));
1678 if (T->getVT().getSizeInBits() == NumBits) {
1679 Src = Val.getOperand(0);
1680 return true;
1681 }
1682 }
1683 break;
1684 case ISD::AND: {
1685 // Check if this is an AND with NumBits of lower bits set to 1.
1686 uint64_t Mask = (1ULL << NumBits) - 1;
1687 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val.getOperand(0))) {
1688 if (C->getZExtValue() == Mask) {
1689 Src = Val.getOperand(1);
1690 return true;
1691 }
1692 }
1693 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val.getOperand(1))) {
1694 if (C->getZExtValue() == Mask) {
1695 Src = Val.getOperand(0);
1696 return true;
1697 }
1698 }
1699 break;
1700 }
1701 case ISD::OR:
1702 case ISD::XOR: {
1703 // OR/XOR with the lower NumBits bits set to 0.
1704 uint64_t Mask = (1ULL << NumBits) - 1;
1705 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val.getOperand(0))) {
1706 if ((C->getZExtValue() & Mask) == 0) {
1707 Src = Val.getOperand(1);
1708 return true;
1709 }
1710 }
1711 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val.getOperand(1))) {
1712 if ((C->getZExtValue() & Mask) == 0) {
1713 Src = Val.getOperand(0);
1714 return true;
1715 }
1716 }
1717 break;
1718 }
1719 default:
1720 break;
1721 }
1722 return false;
1723}
1724
1725bool HexagonDAGToDAGISel::isAlignedMemNode(const MemSDNode *N) const {
1726 return N->getAlign().value() >= N->getMemoryVT().getStoreSize();
1727}
1728
1729bool HexagonDAGToDAGISel::isSmallStackStore(const StoreSDNode *N) const {
1730 unsigned StackSize = MF->getFrameInfo().estimateStackSize(*MF);
1731 switch (N->getMemoryVT().getStoreSize()) {
1732 case 1:
1733 return StackSize <= 56; // 1*2^6 - 8
1734 case 2:
1735 return StackSize <= 120; // 2*2^6 - 8
1736 case 4:
1737 return StackSize <= 248; // 4*2^6 - 8
1738 default:
1739 return false;
1740 }
1741}
1742
1743// Return true when the given node fits in a positive half word.
1744bool HexagonDAGToDAGISel::isPositiveHalfWord(const SDNode *N) const {
1745 if (const ConstantSDNode *CN = dyn_cast<const ConstantSDNode>(N)) {
1746 int64_t V = CN->getSExtValue();
1747 return V > 0 && isInt<16>(V);
1748 }
1749 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
1750 const VTSDNode *VN = dyn_cast<const VTSDNode>(N->getOperand(1));
1751 return VN->getVT().getSizeInBits() <= 16;
1752 }
1753 return false;
1754}
1755
1756bool HexagonDAGToDAGISel::hasOneUse(const SDNode *N) const {
1757 return !CheckSingleUse || N->hasOneUse();
1758}
1759
1760////////////////////////////////////////////////////////////////////////////////
1761// Rebalancing of address calculation trees
1762
1763static bool isOpcodeHandled(const SDNode *N) {
1764 switch (N->getOpcode()) {
1765 case ISD::ADD:
1766 case ISD::MUL:
1767 return true;
1768 case ISD::SHL:
1769 // We only handle constant shifts because these can be easily flattened
1770 // into multiplications by 2^Op1.
1771 return isa<ConstantSDNode>(N->getOperand(1).getNode());
1772 default:
1773 return false;
1774 }
1775}
1776
1777/// Return the weight of an SDNode
1778int HexagonDAGToDAGISel::getWeight(SDNode *N) {
1779 if (!isOpcodeHandled(N))
1780 return 1;
1781 assert(RootWeights.count(N) && "Cannot get weight of unseen root!");
1782 assert(RootWeights[N] != -1 && "Cannot get weight of unvisited root!");
1783 assert(RootWeights[N] != -2 && "Cannot get weight of RAWU'd root!");
1784 return RootWeights[N];
1785}
1786
1787int HexagonDAGToDAGISel::getHeight(SDNode *N) {
1788 if (!isOpcodeHandled(N))
1789 return 0;
1790 assert(RootWeights.count(N) && RootWeights[N] >= 0 &&
1791 "Cannot query height of unvisited/RAUW'd node!");
1792 return RootHeights[N];
1793}
1794
1795namespace {
1796struct WeightedLeaf {
1797 SDValue Value;
1798 int Weight;
1799 int InsertionOrder;
1800
1801 WeightedLeaf() {}
1802
1803 WeightedLeaf(SDValue Value, int Weight, int InsertionOrder) :
1804 Value(Value), Weight(Weight), InsertionOrder(InsertionOrder) {
1805 assert(Weight >= 0 && "Weight must be >= 0");
1806 }
1807
1808 static bool Compare(const WeightedLeaf &A, const WeightedLeaf &B) {
1809 assert(A.Value.getNode() && B.Value.getNode());
1810 return A.Weight == B.Weight ?
1811 (A.InsertionOrder > B.InsertionOrder) :
1812 (A.Weight > B.Weight);
1813 }
1814};
1815
1816/// A specialized priority queue for WeigthedLeaves. It automatically folds
1817/// constants and allows removal of non-top elements while maintaining the
1818/// priority order.
1819class LeafPrioQueue {
1821 bool HaveConst;
1822 WeightedLeaf ConstElt;
1823 unsigned Opcode;
1824
1825public:
1826 bool empty() {
1827 return (!HaveConst && Q.empty());
1828 }
1829
1830 size_t size() {
1831 return Q.size() + HaveConst;
1832 }
1833
1834 bool hasConst() {
1835 return HaveConst;
1836 }
1837
1838 const WeightedLeaf &top() {
1839 if (HaveConst)
1840 return ConstElt;
1841 return Q.front();
1842 }
1843
1844 WeightedLeaf pop() {
1845 if (HaveConst) {
1846 HaveConst = false;
1847 return ConstElt;
1848 }
1849 std::pop_heap(Q.begin(), Q.end(), WeightedLeaf::Compare);
1850 return Q.pop_back_val();
1851 }
1852
1853 void push(WeightedLeaf L, bool SeparateConst=true) {
1854 if (!HaveConst && SeparateConst && isa<ConstantSDNode>(L.Value)) {
1855 if (Opcode == ISD::MUL &&
1856 cast<ConstantSDNode>(L.Value)->getSExtValue() == 1)
1857 return;
1858 if (Opcode == ISD::ADD &&
1859 cast<ConstantSDNode>(L.Value)->getSExtValue() == 0)
1860 return;
1861
1862 HaveConst = true;
1863 ConstElt = L;
1864 } else {
1865 Q.push_back(L);
1866 std::push_heap(Q.begin(), Q.end(), WeightedLeaf::Compare);
1867 }
1868 }
1869
1870 /// Push L to the bottom of the queue regardless of its weight. If L is
1871 /// constant, it will not be folded with other constants in the queue.
1872 void pushToBottom(WeightedLeaf L) {
1873 L.Weight = 1000;
1874 push(L, false);
1875 }
1876
1877 /// Search for a SHL(x, [<=MaxAmount]) subtree in the queue, return the one of
1878 /// lowest weight and remove it from the queue.
1879 WeightedLeaf findSHL(uint64_t MaxAmount);
1880
1881 WeightedLeaf findMULbyConst();
1882
1883 LeafPrioQueue(unsigned Opcode) :
1884 HaveConst(false), Opcode(Opcode) { }
1885};
1886} // end anonymous namespace
1887
1888WeightedLeaf LeafPrioQueue::findSHL(uint64_t MaxAmount) {
1889 int ResultPos;
1890 WeightedLeaf Result;
1891
1892 for (int Pos = 0, End = Q.size(); Pos != End; ++Pos) {
1893 const WeightedLeaf &L = Q[Pos];
1894 const SDValue &Val = L.Value;
1895 if (Val.getOpcode() != ISD::SHL ||
1896 !isa<ConstantSDNode>(Val.getOperand(1)) ||
1897 Val.getConstantOperandVal(1) > MaxAmount)
1898 continue;
1899 if (!Result.Value.getNode() || Result.Weight > L.Weight ||
1900 (Result.Weight == L.Weight && Result.InsertionOrder > L.InsertionOrder))
1901 {
1902 Result = L;
1903 ResultPos = Pos;
1904 }
1905 }
1906
1907 if (Result.Value.getNode()) {
1908 Q.erase(&Q[ResultPos]);
1909 std::make_heap(Q.begin(), Q.end(), WeightedLeaf::Compare);
1910 }
1911
1912 return Result;
1913}
1914
1915WeightedLeaf LeafPrioQueue::findMULbyConst() {
1916 int ResultPos;
1917 WeightedLeaf Result;
1918
1919 for (int Pos = 0, End = Q.size(); Pos != End; ++Pos) {
1920 const WeightedLeaf &L = Q[Pos];
1921 const SDValue &Val = L.Value;
1922 if (Val.getOpcode() != ISD::MUL ||
1923 !isa<ConstantSDNode>(Val.getOperand(1)) ||
1924 Val.getConstantOperandVal(1) > 127)
1925 continue;
1926 if (!Result.Value.getNode() || Result.Weight > L.Weight ||
1927 (Result.Weight == L.Weight && Result.InsertionOrder > L.InsertionOrder))
1928 {
1929 Result = L;
1930 ResultPos = Pos;
1931 }
1932 }
1933
1934 if (Result.Value.getNode()) {
1935 Q.erase(&Q[ResultPos]);
1936 std::make_heap(Q.begin(), Q.end(), WeightedLeaf::Compare);
1937 }
1938
1939 return Result;
1940}
1941
1942SDValue HexagonDAGToDAGISel::getMultiplierForSHL(SDNode *N) {
1943 uint64_t MulFactor = 1ull << N->getConstantOperandVal(1);
1944 return CurDAG->getConstant(MulFactor, SDLoc(N),
1945 N->getOperand(1).getValueType());
1946}
1947
1948/// @returns the value x for which 2^x is a factor of Val
1949static unsigned getPowerOf2Factor(SDValue Val) {
1950 if (Val.getOpcode() == ISD::MUL) {
1951 unsigned MaxFactor = 0;
1952 for (int i = 0; i < 2; ++i) {
1953 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val.getOperand(i));
1954 if (!C)
1955 continue;
1956 const APInt &CInt = C->getAPIntValue();
1957 if (CInt.getBoolValue())
1958 MaxFactor = CInt.countr_zero();
1959 }
1960 return MaxFactor;
1961 }
1962 if (Val.getOpcode() == ISD::SHL) {
1963 if (!isa<ConstantSDNode>(Val.getOperand(1).getNode()))
1964 return 0;
1965 return (unsigned) Val.getConstantOperandVal(1);
1966 }
1967
1968 return 0;
1969}
1970
1971/// @returns true if V>>Amount will eliminate V's operation on its child
1972static bool willShiftRightEliminate(SDValue V, unsigned Amount) {
1973 if (V.getOpcode() == ISD::MUL) {
1974 SDValue Ops[] = { V.getOperand(0), V.getOperand(1) };
1975 for (int i = 0; i < 2; ++i)
1976 if (isa<ConstantSDNode>(Ops[i].getNode()) &&
1977 V.getConstantOperandVal(i) % (1ULL << Amount) == 0) {
1978 uint64_t NewConst = V.getConstantOperandVal(i) >> Amount;
1979 return (NewConst == 1);
1980 }
1981 } else if (V.getOpcode() == ISD::SHL) {
1982 return (Amount == V.getConstantOperandVal(1));
1983 }
1984
1985 return false;
1986}
1987
1988SDValue HexagonDAGToDAGISel::factorOutPowerOf2(SDValue V, unsigned Power) {
1989 SDValue Ops[] = { V.getOperand(0), V.getOperand(1) };
1990 if (V.getOpcode() == ISD::MUL) {
1991 for (int i=0; i < 2; ++i) {
1992 if (isa<ConstantSDNode>(Ops[i].getNode()) &&
1993 V.getConstantOperandVal(i) % ((uint64_t)1 << Power) == 0) {
1994 uint64_t NewConst = V.getConstantOperandVal(i) >> Power;
1995 if (NewConst == 1)
1996 return Ops[!i];
1997 Ops[i] = CurDAG->getConstant(NewConst,
1998 SDLoc(V), V.getValueType());
1999 break;
2000 }
2001 }
2002 } else if (V.getOpcode() == ISD::SHL) {
2003 uint64_t ShiftAmount = V.getConstantOperandVal(1);
2004 if (ShiftAmount == Power)
2005 return Ops[0];
2006 Ops[1] = CurDAG->getConstant(ShiftAmount - Power,
2007 SDLoc(V), V.getValueType());
2008 }
2009
2010 return CurDAG->getNode(V.getOpcode(), SDLoc(V), V.getValueType(), Ops);
2011}
2012
2013static bool isTargetConstant(const SDValue &V) {
2014 return V.getOpcode() == HexagonISD::CONST32 ||
2015 V.getOpcode() == HexagonISD::CONST32_GP;
2016}
2017
2018unsigned HexagonDAGToDAGISel::getUsesInFunction(const Value *V) {
2019 if (GAUsesInFunction.count(V))
2020 return GAUsesInFunction[V];
2021
2022 unsigned Result = 0;
2023 const Function &CurF = CurDAG->getMachineFunction().getFunction();
2024 for (const User *U : V->users()) {
2025 if (isa<Instruction>(U) &&
2026 cast<Instruction>(U)->getParent()->getParent() == &CurF)
2027 ++Result;
2028 }
2029
2030 GAUsesInFunction[V] = Result;
2031
2032 return Result;
2033}
2034
2035/// Note - After calling this, N may be dead. It may have been replaced by a
2036/// new node, so always use the returned value in place of N.
2037///
2038/// @returns The SDValue taking the place of N (which could be N if it is
2039/// unchanged)
2040SDValue HexagonDAGToDAGISel::balanceSubTree(SDNode *N, bool TopLevel) {
2041 assert(RootWeights.count(N) && "Cannot balance non-root node.");
2042 assert(RootWeights[N] != -2 && "This node was RAUW'd!");
2043 assert(!TopLevel || N->getOpcode() == ISD::ADD);
2044
2045 // Return early if this node was already visited
2046 if (RootWeights[N] != -1)
2047 return SDValue(N, 0);
2048
2050
2051 SDValue Op0 = N->getOperand(0);
2052 SDValue Op1 = N->getOperand(1);
2053
2054 // Return early if the operands will remain unchanged or are all roots
2055 if ((!isOpcodeHandled(Op0.getNode()) || RootWeights.count(Op0.getNode())) &&
2056 (!isOpcodeHandled(Op1.getNode()) || RootWeights.count(Op1.getNode()))) {
2057 SDNode *Op0N = Op0.getNode();
2058 int Weight;
2059 if (isOpcodeHandled(Op0N) && RootWeights[Op0N] == -1) {
2060 Weight = getWeight(balanceSubTree(Op0N).getNode());
2061 // Weight = calculateWeight(Op0N);
2062 } else
2063 Weight = getWeight(Op0N);
2064
2065 SDNode *Op1N = N->getOperand(1).getNode(); // Op1 may have been RAUWd
2066 if (isOpcodeHandled(Op1N) && RootWeights[Op1N] == -1) {
2067 Weight += getWeight(balanceSubTree(Op1N).getNode());
2068 // Weight += calculateWeight(Op1N);
2069 } else
2070 Weight += getWeight(Op1N);
2071
2072 RootWeights[N] = Weight;
2073 RootHeights[N] = std::max(getHeight(N->getOperand(0).getNode()),
2074 getHeight(N->getOperand(1).getNode())) + 1;
2075
2076 LLVM_DEBUG(dbgs() << "--> No need to balance root (Weight=" << Weight
2077 << " Height=" << RootHeights[N] << "): ");
2078 LLVM_DEBUG(N->dump(CurDAG));
2079
2080 return SDValue(N, 0);
2081 }
2082
2083 LLVM_DEBUG(dbgs() << "** Balancing root node: ");
2084 LLVM_DEBUG(N->dump(CurDAG));
2085
2086 unsigned NOpcode = N->getOpcode();
2087
2088 LeafPrioQueue Leaves(NOpcode);
2089 SmallVector<SDValue, 4> Worklist;
2090 Worklist.push_back(SDValue(N, 0));
2091
2092 // SHL nodes will be converted to MUL nodes
2093 if (NOpcode == ISD::SHL)
2094 NOpcode = ISD::MUL;
2095
2096 bool CanFactorize = false;
2097 WeightedLeaf Mul1, Mul2;
2098 unsigned MaxPowerOf2 = 0;
2099 WeightedLeaf GA;
2100
2101 // Do not try to factor out a shift if there is already a shift at the tip of
2102 // the tree.
2103 bool HaveTopLevelShift = false;
2104 if (TopLevel &&
2105 ((isOpcodeHandled(Op0.getNode()) && Op0.getOpcode() == ISD::SHL &&
2106 Op0.getConstantOperandVal(1) < 4) ||
2107 (isOpcodeHandled(Op1.getNode()) && Op1.getOpcode() == ISD::SHL &&
2108 Op1.getConstantOperandVal(1) < 4)))
2109 HaveTopLevelShift = true;
2110
2111 // Flatten the subtree into an ordered list of leaves; at the same time
2112 // determine whether the tree is already balanced.
2113 int InsertionOrder = 0;
2114 SmallDenseMap<SDValue, int> NodeHeights;
2115 bool Imbalanced = false;
2116 int CurrentWeight = 0;
2117 while (!Worklist.empty()) {
2118 SDValue Child = Worklist.pop_back_val();
2119
2120 if (Child.getNode() != N && RootWeights.count(Child.getNode())) {
2121 // CASE 1: Child is a root note
2122
2123 int Weight = RootWeights[Child.getNode()];
2124 if (Weight == -1) {
2125 Child = balanceSubTree(Child.getNode());
2126 // calculateWeight(Child.getNode());
2127 Weight = getWeight(Child.getNode());
2128 } else if (Weight == -2) {
2129 // Whoops, this node was RAUWd by one of the balanceSubTree calls we
2130 // made. Our worklist isn't up to date anymore.
2131 // Restart the whole process.
2132 LLVM_DEBUG(dbgs() << "--> Subtree was RAUWd. Restarting...\n");
2133 return balanceSubTree(N, TopLevel);
2134 }
2135
2136 NodeHeights[Child] = 1;
2137 CurrentWeight += Weight;
2138
2139 unsigned PowerOf2;
2140 if (TopLevel && !CanFactorize && !HaveTopLevelShift &&
2141 (Child.getOpcode() == ISD::MUL || Child.getOpcode() == ISD::SHL) &&
2142 Child.hasOneUse() && (PowerOf2 = getPowerOf2Factor(Child))) {
2143 // Try to identify two factorizable MUL/SHL children greedily. Leave
2144 // them out of the priority queue for now so we can deal with them
2145 // after.
2146 if (!Mul1.Value.getNode()) {
2147 Mul1 = WeightedLeaf(Child, Weight, InsertionOrder++);
2148 MaxPowerOf2 = PowerOf2;
2149 } else {
2150 Mul2 = WeightedLeaf(Child, Weight, InsertionOrder++);
2151 MaxPowerOf2 = std::min(MaxPowerOf2, PowerOf2);
2152
2153 // Our addressing modes can only shift by a maximum of 3
2154 if (MaxPowerOf2 > 3)
2155 MaxPowerOf2 = 3;
2156
2157 CanFactorize = true;
2158 }
2159 } else
2160 Leaves.push(WeightedLeaf(Child, Weight, InsertionOrder++));
2161 } else if (!isOpcodeHandled(Child.getNode())) {
2162 // CASE 2: Child is an unhandled kind of node (e.g. constant)
2163 int Weight = getWeight(Child.getNode());
2164
2165 NodeHeights[Child] = getHeight(Child.getNode());
2166 CurrentWeight += Weight;
2167
2168 if (isTargetConstant(Child) && !GA.Value.getNode())
2169 GA = WeightedLeaf(Child, Weight, InsertionOrder++);
2170 else
2171 Leaves.push(WeightedLeaf(Child, Weight, InsertionOrder++));
2172 } else {
2173 // CASE 3: Child is a subtree of same opcode
2174 // Visit children first, then flatten.
2175 unsigned ChildOpcode = Child.getOpcode();
2176 assert(ChildOpcode == NOpcode ||
2177 (NOpcode == ISD::MUL && ChildOpcode == ISD::SHL));
2178
2179 // Convert SHL to MUL
2180 SDValue Op1;
2181 if (ChildOpcode == ISD::SHL)
2182 Op1 = getMultiplierForSHL(Child.getNode());
2183 else
2184 Op1 = Child->getOperand(1);
2185
2186 if (!NodeHeights.count(Op1) || !NodeHeights.count(Child->getOperand(0))) {
2187 assert(!NodeHeights.count(Child) && "Parent visited before children?");
2188 // Visit children first, then re-visit this node
2189 Worklist.push_back(Child);
2190 Worklist.push_back(Op1);
2191 Worklist.push_back(Child->getOperand(0));
2192 } else {
2193 // Back at this node after visiting the children
2194 if (std::abs(NodeHeights[Op1] - NodeHeights[Child->getOperand(0)]) > 1)
2195 Imbalanced = true;
2196
2197 NodeHeights[Child] = std::max(NodeHeights[Op1],
2198 NodeHeights[Child->getOperand(0)]) + 1;
2199 }
2200 }
2201 }
2202
2203 LLVM_DEBUG(dbgs() << "--> Current height=" << NodeHeights[SDValue(N, 0)]
2204 << " weight=" << CurrentWeight
2205 << " imbalanced=" << Imbalanced << "\n");
2206
2207 // Transform MUL(x, C * 2^Y) + SHL(z, Y) -> SHL(ADD(MUL(x, C), z), Y)
2208 // This factors out a shift in order to match memw(a<<Y+b).
2209 if (CanFactorize && (willShiftRightEliminate(Mul1.Value, MaxPowerOf2) ||
2210 willShiftRightEliminate(Mul2.Value, MaxPowerOf2))) {
2211 LLVM_DEBUG(dbgs() << "--> Found common factor for two MUL children!\n");
2212 int Weight = Mul1.Weight + Mul2.Weight;
2213 int Height = std::max(NodeHeights[Mul1.Value], NodeHeights[Mul2.Value]) + 1;
2214 SDValue Mul1Factored = factorOutPowerOf2(Mul1.Value, MaxPowerOf2);
2215 SDValue Mul2Factored = factorOutPowerOf2(Mul2.Value, MaxPowerOf2);
2216 SDValue Sum = CurDAG->getNode(ISD::ADD, SDLoc(N), Mul1.Value.getValueType(),
2217 Mul1Factored, Mul2Factored);
2218 SDValue Const = CurDAG->getConstant(MaxPowerOf2, SDLoc(N),
2219 Mul1.Value.getValueType());
2220 SDValue New = CurDAG->getNode(ISD::SHL, SDLoc(N), Mul1.Value.getValueType(),
2221 Sum, Const);
2222 NodeHeights[New] = Height;
2223 Leaves.push(WeightedLeaf(New, Weight, Mul1.InsertionOrder));
2224 } else if (Mul1.Value.getNode()) {
2225 // We failed to factorize two MULs, so now the Muls are left outside the
2226 // queue... add them back.
2227 Leaves.push(Mul1);
2228 if (Mul2.Value.getNode())
2229 Leaves.push(Mul2);
2230 CanFactorize = false;
2231 }
2232
2233 // Combine GA + Constant -> GA+Offset, but only if GA is not used elsewhere
2234 // and the root node itself is not used more than twice. This reduces the
2235 // amount of additional constant extenders introduced by this optimization.
2236 bool CombinedGA = false;
2237 if (NOpcode == ISD::ADD && GA.Value.getNode() && Leaves.hasConst() &&
2238 GA.Value.hasOneUse() && N->use_size() < 3) {
2239 GlobalAddressSDNode *GANode =
2240 cast<GlobalAddressSDNode>(GA.Value.getOperand(0));
2241 ConstantSDNode *Offset = cast<ConstantSDNode>(Leaves.top().Value);
2242
2243 if (getUsesInFunction(GANode->getGlobal()) == 1 && Offset->hasOneUse() &&
2244 getTargetLowering()->isOffsetFoldingLegal(GANode)) {
2245 LLVM_DEBUG(dbgs() << "--> Combining GA and offset ("
2246 << Offset->getSExtValue() << "): ");
2247 LLVM_DEBUG(GANode->dump(CurDAG));
2248
2249 SDValue NewTGA =
2250 CurDAG->getTargetGlobalAddress(GANode->getGlobal(), SDLoc(GA.Value),
2251 GANode->getValueType(0),
2252 GANode->getOffset() + (uint64_t)Offset->getSExtValue());
2253 GA.Value = CurDAG->getNode(GA.Value.getOpcode(), SDLoc(GA.Value),
2254 GA.Value.getValueType(), NewTGA);
2255 GA.Weight += Leaves.top().Weight;
2256
2257 NodeHeights[GA.Value] = getHeight(GA.Value.getNode());
2258 CombinedGA = true;
2259
2260 Leaves.pop(); // Remove the offset constant from the queue
2261 }
2262 }
2263
2264 if ((RebalanceOnlyForOptimizations && !CanFactorize && !CombinedGA) ||
2265 (RebalanceOnlyImbalancedTrees && !Imbalanced)) {
2266 RootWeights[N] = CurrentWeight;
2267 RootHeights[N] = NodeHeights[SDValue(N, 0)];
2268
2269 return SDValue(N, 0);
2270 }
2271
2272 // Combine GA + SHL(x, C<=31) so we will match Rx=add(#u8,asl(Rx,#U5))
2273 if (NOpcode == ISD::ADD && GA.Value.getNode()) {
2274 WeightedLeaf SHL = Leaves.findSHL(31);
2275 if (SHL.Value.getNode()) {
2276 int Height = std::max(NodeHeights[GA.Value], NodeHeights[SHL.Value]) + 1;
2277 GA.Value = CurDAG->getNode(ISD::ADD, SDLoc(GA.Value),
2278 GA.Value.getValueType(),
2279 GA.Value, SHL.Value);
2280 GA.Weight = SHL.Weight; // Specifically ignore the GA weight here
2281 NodeHeights[GA.Value] = Height;
2282 }
2283 }
2284
2285 if (GA.Value.getNode())
2286 Leaves.push(GA);
2287
2288 // If this is the top level and we haven't factored out a shift, we should try
2289 // to move a constant to the bottom to match addressing modes like memw(rX+C)
2290 if (TopLevel && !CanFactorize && Leaves.hasConst()) {
2291 LLVM_DEBUG(dbgs() << "--> Pushing constant to tip of tree.");
2292 Leaves.pushToBottom(Leaves.pop());
2293 }
2294
2295 const DataLayout &DL = CurDAG->getDataLayout();
2297
2298 // Rebuild the tree using Huffman's algorithm
2299 while (Leaves.size() > 1) {
2300 WeightedLeaf L0 = Leaves.pop();
2301
2302 // See whether we can grab a MUL to form an add(Rx,mpyi(Ry,#u6)),
2303 // otherwise just get the next leaf
2304 WeightedLeaf L1 = Leaves.findMULbyConst();
2305 if (!L1.Value.getNode())
2306 L1 = Leaves.pop();
2307
2308 assert(L0.Weight <= L1.Weight && "Priority queue is broken!");
2309
2310 SDValue V0 = L0.Value;
2311 int V0Weight = L0.Weight;
2312 SDValue V1 = L1.Value;
2313 int V1Weight = L1.Weight;
2314
2315 // Make sure that none of these nodes have been RAUW'd
2316 if ((RootWeights.count(V0.getNode()) && RootWeights[V0.getNode()] == -2) ||
2317 (RootWeights.count(V1.getNode()) && RootWeights[V1.getNode()] == -2)) {
2318 LLVM_DEBUG(dbgs() << "--> Subtree was RAUWd. Restarting...\n");
2319 return balanceSubTree(N, TopLevel);
2320 }
2321
2322 ConstantSDNode *V0C = dyn_cast<ConstantSDNode>(V0);
2323 ConstantSDNode *V1C = dyn_cast<ConstantSDNode>(V1);
2324 EVT VT = N->getValueType(0);
2325 SDValue NewNode;
2326
2327 if (V0C && !V1C) {
2328 std::swap(V0, V1);
2329 std::swap(V0C, V1C);
2330 }
2331
2332 // Calculate height of this node
2333 assert(NodeHeights.count(V0) && NodeHeights.count(V1) &&
2334 "Children must have been visited before re-combining them!");
2335 int Height = std::max(NodeHeights[V0], NodeHeights[V1]) + 1;
2336
2337 // Rebuild this node (and restore SHL from MUL if needed)
2338 if (V1C && NOpcode == ISD::MUL && V1C->getAPIntValue().isPowerOf2())
2339 NewNode = CurDAG->getNode(
2340 ISD::SHL, SDLoc(V0), VT, V0,
2342 V1C->getAPIntValue().logBase2(), SDLoc(N),
2344 else
2345 NewNode = CurDAG->getNode(NOpcode, SDLoc(N), VT, V0, V1);
2346
2347 NodeHeights[NewNode] = Height;
2348
2349 int Weight = V0Weight + V1Weight;
2350 Leaves.push(WeightedLeaf(NewNode, Weight, L0.InsertionOrder));
2351
2352 LLVM_DEBUG(dbgs() << "--> Built new node (Weight=" << Weight
2353 << ",Height=" << Height << "):\n");
2354 LLVM_DEBUG(NewNode.dump());
2355 }
2356
2357 assert(Leaves.size() == 1);
2358 SDValue NewRoot = Leaves.top().Value;
2359
2360 assert(NodeHeights.count(NewRoot));
2361 int Height = NodeHeights[NewRoot];
2362
2363 // Restore SHL if we earlier converted it to a MUL
2364 if (NewRoot.getOpcode() == ISD::MUL) {
2365 ConstantSDNode *V1C = dyn_cast<ConstantSDNode>(NewRoot.getOperand(1));
2366 if (V1C && V1C->getAPIntValue().isPowerOf2()) {
2367 EVT VT = NewRoot.getValueType();
2368 SDValue V0 = NewRoot.getOperand(0);
2369 NewRoot = CurDAG->getNode(
2370 ISD::SHL, SDLoc(NewRoot), VT, V0,
2372 V1C->getAPIntValue().logBase2(), SDLoc(NewRoot),
2374 }
2375 }
2376
2377 if (N != NewRoot.getNode()) {
2378 LLVM_DEBUG(dbgs() << "--> Root is now: ");
2379 LLVM_DEBUG(NewRoot.dump());
2380
2381 // Replace all uses of old root by new root
2382 CurDAG->ReplaceAllUsesWith(N, NewRoot.getNode());
2383 // Mark that we have RAUW'd N
2384 RootWeights[N] = -2;
2385 } else {
2386 LLVM_DEBUG(dbgs() << "--> Root unchanged.\n");
2387 }
2388
2389 RootWeights[NewRoot.getNode()] = Leaves.top().Weight;
2390 RootHeights[NewRoot.getNode()] = Height;
2391
2392 return NewRoot;
2393}
2394
2395void HexagonDAGToDAGISel::rebalanceAddressTrees() {
2397 SDNode *N = &Node;
2398 if (N->getOpcode() != ISD::LOAD && N->getOpcode() != ISD::STORE)
2399 continue;
2400
2401 SDValue BasePtr = cast<MemSDNode>(N)->getBasePtr();
2402 if (BasePtr.getOpcode() != ISD::ADD)
2403 continue;
2404
2405 // We've already processed this node
2406 if (RootWeights.count(BasePtr.getNode()))
2407 continue;
2408
2409 LLVM_DEBUG(dbgs() << "** Rebalancing address calculation in node: ");
2410 LLVM_DEBUG(N->dump(CurDAG));
2411
2412 // FindRoots
2413 SmallVector<SDNode *, 4> Worklist;
2414
2415 Worklist.push_back(BasePtr.getOperand(0).getNode());
2416 Worklist.push_back(BasePtr.getOperand(1).getNode());
2417
2418 while (!Worklist.empty()) {
2419 SDNode *N = Worklist.pop_back_val();
2420 unsigned Opcode = N->getOpcode();
2421
2422 if (!isOpcodeHandled(N))
2423 continue;
2424
2425 Worklist.push_back(N->getOperand(0).getNode());
2426 Worklist.push_back(N->getOperand(1).getNode());
2427
2428 // Not a root if it has only one use and same opcode as its parent
2429 if (N->hasOneUse() && Opcode == N->use_begin()->getOpcode())
2430 continue;
2431
2432 // This root node has already been processed
2433 if (RootWeights.count(N))
2434 continue;
2435
2436 RootWeights[N] = -1;
2437 }
2438
2439 // Balance node itself
2440 RootWeights[BasePtr.getNode()] = -1;
2441 SDValue NewBasePtr = balanceSubTree(BasePtr.getNode(), /*TopLevel=*/ true);
2442
2443 if (N->getOpcode() == ISD::LOAD)
2444 N = CurDAG->UpdateNodeOperands(N, N->getOperand(0),
2445 NewBasePtr, N->getOperand(2));
2446 else
2447 N = CurDAG->UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2448 NewBasePtr, N->getOperand(3));
2449
2450 LLVM_DEBUG(dbgs() << "--> Final node: ");
2451 LLVM_DEBUG(N->dump(CurDAG));
2452 }
2453
2455 GAUsesInFunction.clear();
2456 RootHeights.clear();
2457 RootWeights.clear();
2458}
unsigned SubReg
aarch64 promote const
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const LLT S1
static const Function * getParent(const Value *V)
static void push(SmallVectorImpl< uint64_t > &R, StringRef Str)
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition: Debug.h:64
uint64_t Addr
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static bool willShiftRightEliminate(SDValue V, unsigned Amount)
static cl::opt< bool > RebalanceOnlyImbalancedTrees("rebalance-only-imbal", cl::Hidden, cl::init(false), cl::desc("Rebalance address tree only if it is imbalanced"))
static unsigned getPowerOf2Factor(SDValue Val)
static cl::opt< bool > CheckSingleUse("hexagon-isel-su", cl::Hidden, cl::init(true), cl::desc("Enable checking of SDNode's single-use status"))
static cl::opt< bool > EnableAddressRebalancing("isel-rebalance-addr", cl::Hidden, cl::init(true), cl::desc("Rebalance address calculation trees to improve " "instruction selection"))
static bool isMemOPCandidate(SDNode *I, SDNode *U)
#define PASS_NAME
#define DEBUG_TYPE
static bool isTargetConstant(const SDValue &V)
static bool isOpcodeHandled(const SDNode *N)
static cl::opt< bool > RebalanceOnlyForOptimizations("rebalance-only-opt", cl::Hidden, cl::init(false), cl::desc("Rebalance address tree only if this allows optimizations"))
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define T1
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
#define CH(x, y, z)
Definition: SHA256.cpp:34
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Class for arbitrary precision integers.
Definition: APInt.h:76
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1439
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1589
unsigned logBase2() const
Definition: APInt.h:1703
bool getBoolValue() const
Convert APInt to a boolean value.
Definition: APInt.h:449
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
const GlobalValue * getGlobal() const
This class is used to form a handle around another node that is persistent and is updated across invo...
const SDValue & getValue() const
void Select(SDNode *N) override
Main hook for targets to transform nodes into machine nodes.
bool SelectNewCircIntrinsic(SDNode *IntN)
Generate a machine instruction node for the new circular buffer intrinsics.
bool tryLoadOfLoadIntrinsic(LoadSDNode *N)
void SelectIndexedLoad(LoadSDNode *LD, const SDLoc &dl)
MachineSDNode * LoadInstrForLoadIntrinsic(SDNode *IntN)
bool SelectAnyImm2(SDValue &N, SDValue &R)
bool SelectAnyImm(SDValue &N, SDValue &R)
bool SelectAnyImm0(SDValue &N, SDValue &R)
bool SelectAnyImm1(SDValue &N, SDValue &R)
bool DetectUseSxtw(SDValue &N, SDValue &R)
bool SelectBrevLdIntrinsic(SDNode *IntN)
bool SelectAddrFI(SDValue &N, SDValue &R)
SDNode * StoreInstrForLoadIntrinsic(MachineSDNode *LoadN, SDNode *IntN)
bool SelectAddrGP(SDValue &N, SDValue &R)
bool SelectAnyImmediate(SDValue &N, SDValue &R, Align Alignment)
bool SelectGlobalAddress(SDValue &N, SDValue &R, bool UseGP, Align Alignment)
bool SelectAnyImm3(SDValue &N, SDValue &R)
bool SelectAnyInt(SDValue &N, SDValue &R)
bool SelectAddrGA(SDValue &N, SDValue &R)
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Implement addressing mode selection for inline asm expressions.
void SelectIndexedStore(StoreSDNode *ST, const SDLoc &dl)
bool isValidAutoIncImm(const EVT VT, const int Offset) const
Hexagon target-specific information for each MachineFunction.
BitVector getReservedRegs(const MachineFunction &MF) const override
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
const HexagonFrameLowering * getFrameLowering() const override
const HexagonRegisterInfo * getRegisterInfo() const override
bool isHVXVectorType(EVT VecTy, bool IncludeBool=false) const
unsigned getVectorLength() const
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
SimpleValueType SimpleTy
unsigned getVectorNumElements() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getVectorElementType() const
static MVT getIntegerVT(unsigned BitWidth)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Representation of each machine instruction.
Definition: MachineInstr.h:69
A description of a memory reference used in the backend.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isValid() const
Definition: Register.h:116
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
const TargetLowering * TLI
MachineFunction * MF
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
const TargetLowering * getTargetLowering() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:722
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:727
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
iterator_range< allnodes_iterator > allnodes()
Definition: SelectionDAG.h:543
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ilist< SDNode >::size_type allnodes_size() const
Definition: SelectionDAG.h:539
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:799
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
iterator erase(const_iterator CI)
Definition: SmallVector.h:750
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const
Return the type to use for a scalar shift opcode, given the shifted amount type.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Definition: Value.h:74
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FrameIndex
Definition: ISDOpcodes.h:80
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:164
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ ExternalSymbol
Definition: ISDOpcodes.h:83
@ BlockAddress
Definition: ISDOpcodes.h:84
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ AssertZext
Definition: ISDOpcodes.h:62
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1472
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1503
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1689
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
Definition: Alignment.h:145
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:665
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
unsigned M1(unsigned Val)
Definition: VE.h:376
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: bit.h:281
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
FunctionPass * createHexagonISelDag(HexagonTargetMachine &TM, CodeGenOptLevel OptLevel)
createHexagonISelDag - This pass converts a legalized DAG into a Hexagon-specific DAG,...
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
Definition: VE.h:375
@ Default
The result values are uniform if and only if all operands are uniform.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
This class contains a discriminated union of information about pointers in memory operands,...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.