LLVM 17.0.0git
LegalizerHelper.cpp
Go to the documentation of this file.
1//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize
10/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
34#include "llvm/Support/Debug.h"
38#include <numeric>
39#include <optional>
40
41#define DEBUG_TYPE "legalizer"
42
43using namespace llvm;
44using namespace LegalizeActions;
45using namespace MIPatternMatch;
46
47/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
48///
49/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
50/// with any leftover piece as type \p LeftoverTy
51///
52/// Returns -1 in the first element of the pair if the breakdown is not
53/// satisfiable.
54static std::pair<int, int>
55getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
56 assert(!LeftoverTy.isValid() && "this is an out argument");
57
58 unsigned Size = OrigTy.getSizeInBits();
59 unsigned NarrowSize = NarrowTy.getSizeInBits();
60 unsigned NumParts = Size / NarrowSize;
61 unsigned LeftoverSize = Size - NumParts * NarrowSize;
62 assert(Size > NarrowSize);
63
64 if (LeftoverSize == 0)
65 return {NumParts, 0};
66
67 if (NarrowTy.isVector()) {
68 unsigned EltSize = OrigTy.getScalarSizeInBits();
69 if (LeftoverSize % EltSize != 0)
70 return {-1, -1};
71 LeftoverTy = LLT::scalarOrVector(
72 ElementCount::getFixed(LeftoverSize / EltSize), EltSize);
73 } else {
74 LeftoverTy = LLT::scalar(LeftoverSize);
75 }
76
77 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
78 return std::make_pair(NumParts, NumLeftover);
79}
80
82
83 if (!Ty.isScalar())
84 return nullptr;
85
86 switch (Ty.getSizeInBits()) {
87 case 16:
88 return Type::getHalfTy(Ctx);
89 case 32:
90 return Type::getFloatTy(Ctx);
91 case 64:
92 return Type::getDoubleTy(Ctx);
93 case 80:
94 return Type::getX86_FP80Ty(Ctx);
95 case 128:
96 return Type::getFP128Ty(Ctx);
97 default:
98 return nullptr;
99 }
100}
101
103 GISelChangeObserver &Observer,
104 MachineIRBuilder &Builder)
105 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
106 LI(*MF.getSubtarget().getLegalizerInfo()),
107 TLI(*MF.getSubtarget().getTargetLowering()), KB(nullptr) {}
108
110 GISelChangeObserver &Observer,
112 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
113 TLI(*MF.getSubtarget().getTargetLowering()), KB(KB) {}
114
117 LostDebugLocObserver &LocObserver) {
118 LLVM_DEBUG(dbgs() << "Legalizing: " << MI);
119
121
122 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
123 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
124 return LI.legalizeIntrinsic(*this, MI) ? Legalized : UnableToLegalize;
125 auto Step = LI.getAction(MI, MRI);
126 switch (Step.Action) {
127 case Legal:
128 LLVM_DEBUG(dbgs() << ".. Already legal\n");
129 return AlreadyLegal;
130 case Libcall:
131 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
132 return libcall(MI, LocObserver);
133 case NarrowScalar:
134 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
135 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
136 case WidenScalar:
137 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
138 return widenScalar(MI, Step.TypeIdx, Step.NewType);
139 case Bitcast:
140 LLVM_DEBUG(dbgs() << ".. Bitcast type\n");
141 return bitcast(MI, Step.TypeIdx, Step.NewType);
142 case Lower:
143 LLVM_DEBUG(dbgs() << ".. Lower\n");
144 return lower(MI, Step.TypeIdx, Step.NewType);
145 case FewerElements:
146 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
147 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
148 case MoreElements:
149 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
150 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
151 case Custom:
152 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
153 return LI.legalizeCustom(*this, MI) ? Legalized : UnableToLegalize;
154 default:
155 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
156 return UnableToLegalize;
157 }
158}
159
160void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
162 for (int i = 0; i < NumParts; ++i)
164 MIRBuilder.buildUnmerge(VRegs, Reg);
165}
166
167bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
168 LLT MainTy, LLT &LeftoverTy,
170 SmallVectorImpl<Register> &LeftoverRegs) {
171 assert(!LeftoverTy.isValid() && "this is an out argument");
172
173 unsigned RegSize = RegTy.getSizeInBits();
174 unsigned MainSize = MainTy.getSizeInBits();
175 unsigned NumParts = RegSize / MainSize;
176 unsigned LeftoverSize = RegSize - NumParts * MainSize;
177
178 // Use an unmerge when possible.
179 if (LeftoverSize == 0) {
180 for (unsigned I = 0; I < NumParts; ++I)
181 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
182 MIRBuilder.buildUnmerge(VRegs, Reg);
183 return true;
184 }
185
186 // Perform irregular split. Leftover is last element of RegPieces.
187 if (MainTy.isVector()) {
188 SmallVector<Register, 8> RegPieces;
189 extractVectorParts(Reg, MainTy.getNumElements(), RegPieces);
190 for (unsigned i = 0; i < RegPieces.size() - 1; ++i)
191 VRegs.push_back(RegPieces[i]);
192 LeftoverRegs.push_back(RegPieces[RegPieces.size() - 1]);
193 LeftoverTy = MRI.getType(LeftoverRegs[0]);
194 return true;
195 }
196
197 LeftoverTy = LLT::scalar(LeftoverSize);
198 // For irregular sizes, extract the individual parts.
199 for (unsigned I = 0; I != NumParts; ++I) {
200 Register NewReg = MRI.createGenericVirtualRegister(MainTy);
201 VRegs.push_back(NewReg);
202 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
203 }
204
205 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
206 Offset += LeftoverSize) {
207 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
208 LeftoverRegs.push_back(NewReg);
209 MIRBuilder.buildExtract(NewReg, Reg, Offset);
210 }
211
212 return true;
213}
214
215void LegalizerHelper::extractVectorParts(Register Reg, unsigned NumElts,
217 LLT RegTy = MRI.getType(Reg);
218 assert(RegTy.isVector() && "Expected a vector type");
219
220 LLT EltTy = RegTy.getElementType();
221 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy);
222 unsigned RegNumElts = RegTy.getNumElements();
223 unsigned LeftoverNumElts = RegNumElts % NumElts;
224 unsigned NumNarrowTyPieces = RegNumElts / NumElts;
225
226 // Perfect split without leftover
227 if (LeftoverNumElts == 0)
228 return extractParts(Reg, NarrowTy, NumNarrowTyPieces, VRegs);
229
230 // Irregular split. Provide direct access to all elements for artifact
231 // combiner using unmerge to elements. Then build vectors with NumElts
232 // elements. Remaining element(s) will be (used to build vector) Leftover.
234 extractParts(Reg, EltTy, RegNumElts, Elts);
235
236 unsigned Offset = 0;
237 // Requested sub-vectors of NarrowTy.
238 for (unsigned i = 0; i < NumNarrowTyPieces; ++i, Offset += NumElts) {
239 ArrayRef<Register> Pieces(&Elts[Offset], NumElts);
240 VRegs.push_back(MIRBuilder.buildMergeLikeInstr(NarrowTy, Pieces).getReg(0));
241 }
242
243 // Leftover element(s).
244 if (LeftoverNumElts == 1) {
245 VRegs.push_back(Elts[Offset]);
246 } else {
247 LLT LeftoverTy = LLT::fixed_vector(LeftoverNumElts, EltTy);
248 ArrayRef<Register> Pieces(&Elts[Offset], LeftoverNumElts);
249 VRegs.push_back(
250 MIRBuilder.buildMergeLikeInstr(LeftoverTy, Pieces).getReg(0));
251 }
252}
253
254void LegalizerHelper::insertParts(Register DstReg,
255 LLT ResultTy, LLT PartTy,
256 ArrayRef<Register> PartRegs,
257 LLT LeftoverTy,
258 ArrayRef<Register> LeftoverRegs) {
259 if (!LeftoverTy.isValid()) {
260 assert(LeftoverRegs.empty());
261
262 if (!ResultTy.isVector()) {
263 MIRBuilder.buildMergeLikeInstr(DstReg, PartRegs);
264 return;
265 }
266
267 if (PartTy.isVector())
268 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
269 else
270 MIRBuilder.buildBuildVector(DstReg, PartRegs);
271 return;
272 }
273
274 // Merge sub-vectors with different number of elements and insert into DstReg.
275 if (ResultTy.isVector()) {
276 assert(LeftoverRegs.size() == 1 && "Expected one leftover register");
278 for (auto Reg : concat<const Register>(PartRegs, LeftoverRegs))
279 AllRegs.push_back(Reg);
280 return mergeMixedSubvectors(DstReg, AllRegs);
281 }
282
283 SmallVector<Register> GCDRegs;
284 LLT GCDTy = getGCDType(getGCDType(ResultTy, LeftoverTy), PartTy);
285 for (auto PartReg : concat<const Register>(PartRegs, LeftoverRegs))
286 extractGCDType(GCDRegs, GCDTy, PartReg);
287 LLT ResultLCMTy = buildLCMMergePieces(ResultTy, LeftoverTy, GCDTy, GCDRegs);
288 buildWidenedRemergeToDst(DstReg, ResultLCMTy, GCDRegs);
289}
290
291void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts,
292 Register Reg) {
293 LLT Ty = MRI.getType(Reg);
295 extractParts(Reg, Ty.getScalarType(), Ty.getNumElements(), RegElts);
296 Elts.append(RegElts);
297}
298
299/// Merge \p PartRegs with different types into \p DstReg.
300void LegalizerHelper::mergeMixedSubvectors(Register DstReg,
301 ArrayRef<Register> PartRegs) {
303 for (unsigned i = 0; i < PartRegs.size() - 1; ++i)
304 appendVectorElts(AllElts, PartRegs[i]);
305
306 Register Leftover = PartRegs[PartRegs.size() - 1];
307 if (MRI.getType(Leftover).isScalar())
308 AllElts.push_back(Leftover);
309 else
310 appendVectorElts(AllElts, Leftover);
311
312 MIRBuilder.buildMergeLikeInstr(DstReg, AllElts);
313}
314
315/// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
317 const MachineInstr &MI) {
318 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
319
320 const int StartIdx = Regs.size();
321 const int NumResults = MI.getNumOperands() - 1;
322 Regs.resize(Regs.size() + NumResults);
323 for (int I = 0; I != NumResults; ++I)
324 Regs[StartIdx + I] = MI.getOperand(I).getReg();
325}
326
327void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
328 LLT GCDTy, Register SrcReg) {
329 LLT SrcTy = MRI.getType(SrcReg);
330 if (SrcTy == GCDTy) {
331 // If the source already evenly divides the result type, we don't need to do
332 // anything.
333 Parts.push_back(SrcReg);
334 } else {
335 // Need to split into common type sized pieces.
336 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
337 getUnmergeResults(Parts, *Unmerge);
338 }
339}
340
341LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
342 LLT NarrowTy, Register SrcReg) {
343 LLT SrcTy = MRI.getType(SrcReg);
344 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
345 extractGCDType(Parts, GCDTy, SrcReg);
346 return GCDTy;
347}
348
349LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
351 unsigned PadStrategy) {
352 LLT LCMTy = getLCMType(DstTy, NarrowTy);
353
354 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
355 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
356 int NumOrigSrc = VRegs.size();
357
358 Register PadReg;
359
360 // Get a value we can use to pad the source value if the sources won't evenly
361 // cover the result type.
362 if (NumOrigSrc < NumParts * NumSubParts) {
363 if (PadStrategy == TargetOpcode::G_ZEXT)
364 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0);
365 else if (PadStrategy == TargetOpcode::G_ANYEXT)
366 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
367 else {
368 assert(PadStrategy == TargetOpcode::G_SEXT);
369
370 // Shift the sign bit of the low register through the high register.
371 auto ShiftAmt =
373 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0);
374 }
375 }
376
377 // Registers for the final merge to be produced.
378 SmallVector<Register, 4> Remerge(NumParts);
379
380 // Registers needed for intermediate merges, which will be merged into a
381 // source for Remerge.
382 SmallVector<Register, 4> SubMerge(NumSubParts);
383
384 // Once we've fully read off the end of the original source bits, we can reuse
385 // the same high bits for remaining padding elements.
386 Register AllPadReg;
387
388 // Build merges to the LCM type to cover the original result type.
389 for (int I = 0; I != NumParts; ++I) {
390 bool AllMergePartsArePadding = true;
391
392 // Build the requested merges to the requested type.
393 for (int J = 0; J != NumSubParts; ++J) {
394 int Idx = I * NumSubParts + J;
395 if (Idx >= NumOrigSrc) {
396 SubMerge[J] = PadReg;
397 continue;
398 }
399
400 SubMerge[J] = VRegs[Idx];
401
402 // There are meaningful bits here we can't reuse later.
403 AllMergePartsArePadding = false;
404 }
405
406 // If we've filled up a complete piece with padding bits, we can directly
407 // emit the natural sized constant if applicable, rather than a merge of
408 // smaller constants.
409 if (AllMergePartsArePadding && !AllPadReg) {
410 if (PadStrategy == TargetOpcode::G_ANYEXT)
411 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0);
412 else if (PadStrategy == TargetOpcode::G_ZEXT)
413 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0);
414
415 // If this is a sign extension, we can't materialize a trivial constant
416 // with the right type and have to produce a merge.
417 }
418
419 if (AllPadReg) {
420 // Avoid creating additional instructions if we're just adding additional
421 // copies of padding bits.
422 Remerge[I] = AllPadReg;
423 continue;
424 }
425
426 if (NumSubParts == 1)
427 Remerge[I] = SubMerge[0];
428 else
429 Remerge[I] = MIRBuilder.buildMergeLikeInstr(NarrowTy, SubMerge).getReg(0);
430
431 // In the sign extend padding case, re-use the first all-signbit merge.
432 if (AllMergePartsArePadding && !AllPadReg)
433 AllPadReg = Remerge[I];
434 }
435
436 VRegs = std::move(Remerge);
437 return LCMTy;
438}
439
440void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
441 ArrayRef<Register> RemergeRegs) {
442 LLT DstTy = MRI.getType(DstReg);
443
444 // Create the merge to the widened source, and extract the relevant bits into
445 // the result.
446
447 if (DstTy == LCMTy) {
448 MIRBuilder.buildMergeLikeInstr(DstReg, RemergeRegs);
449 return;
450 }
451
452 auto Remerge = MIRBuilder.buildMergeLikeInstr(LCMTy, RemergeRegs);
453 if (DstTy.isScalar() && LCMTy.isScalar()) {
454 MIRBuilder.buildTrunc(DstReg, Remerge);
455 return;
456 }
457
458 if (LCMTy.isVector()) {
459 unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
460 SmallVector<Register, 8> UnmergeDefs(NumDefs);
461 UnmergeDefs[0] = DstReg;
462 for (unsigned I = 1; I != NumDefs; ++I)
463 UnmergeDefs[I] = MRI.createGenericVirtualRegister(DstTy);
464
465 MIRBuilder.buildUnmerge(UnmergeDefs,
466 MIRBuilder.buildMergeLikeInstr(LCMTy, RemergeRegs));
467 return;
468 }
469
470 llvm_unreachable("unhandled case");
471}
472
473static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
474#define RTLIBCASE_INT(LibcallPrefix) \
475 do { \
476 switch (Size) { \
477 case 32: \
478 return RTLIB::LibcallPrefix##32; \
479 case 64: \
480 return RTLIB::LibcallPrefix##64; \
481 case 128: \
482 return RTLIB::LibcallPrefix##128; \
483 default: \
484 llvm_unreachable("unexpected size"); \
485 } \
486 } while (0)
487
488#define RTLIBCASE(LibcallPrefix) \
489 do { \
490 switch (Size) { \
491 case 32: \
492 return RTLIB::LibcallPrefix##32; \
493 case 64: \
494 return RTLIB::LibcallPrefix##64; \
495 case 80: \
496 return RTLIB::LibcallPrefix##80; \
497 case 128: \
498 return RTLIB::LibcallPrefix##128; \
499 default: \
500 llvm_unreachable("unexpected size"); \
501 } \
502 } while (0)
503
504 switch (Opcode) {
505 case TargetOpcode::G_MUL:
506 RTLIBCASE_INT(MUL_I);
507 case TargetOpcode::G_SDIV:
508 RTLIBCASE_INT(SDIV_I);
509 case TargetOpcode::G_UDIV:
510 RTLIBCASE_INT(UDIV_I);
511 case TargetOpcode::G_SREM:
512 RTLIBCASE_INT(SREM_I);
513 case TargetOpcode::G_UREM:
514 RTLIBCASE_INT(UREM_I);
515 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
516 RTLIBCASE_INT(CTLZ_I);
517 case TargetOpcode::G_FADD:
518 RTLIBCASE(ADD_F);
519 case TargetOpcode::G_FSUB:
520 RTLIBCASE(SUB_F);
521 case TargetOpcode::G_FMUL:
522 RTLIBCASE(MUL_F);
523 case TargetOpcode::G_FDIV:
524 RTLIBCASE(DIV_F);
525 case TargetOpcode::G_FEXP:
526 RTLIBCASE(EXP_F);
527 case TargetOpcode::G_FEXP2:
528 RTLIBCASE(EXP2_F);
529 case TargetOpcode::G_FREM:
530 RTLIBCASE(REM_F);
531 case TargetOpcode::G_FPOW:
532 RTLIBCASE(POW_F);
533 case TargetOpcode::G_FMA:
534 RTLIBCASE(FMA_F);
535 case TargetOpcode::G_FSIN:
536 RTLIBCASE(SIN_F);
537 case TargetOpcode::G_FCOS:
538 RTLIBCASE(COS_F);
539 case TargetOpcode::G_FLOG10:
540 RTLIBCASE(LOG10_F);
541 case TargetOpcode::G_FLOG:
542 RTLIBCASE(LOG_F);
543 case TargetOpcode::G_FLOG2:
544 RTLIBCASE(LOG2_F);
545 case TargetOpcode::G_FCEIL:
546 RTLIBCASE(CEIL_F);
547 case TargetOpcode::G_FFLOOR:
548 RTLIBCASE(FLOOR_F);
549 case TargetOpcode::G_FMINNUM:
550 RTLIBCASE(FMIN_F);
551 case TargetOpcode::G_FMAXNUM:
552 RTLIBCASE(FMAX_F);
553 case TargetOpcode::G_FSQRT:
554 RTLIBCASE(SQRT_F);
555 case TargetOpcode::G_FRINT:
556 RTLIBCASE(RINT_F);
557 case TargetOpcode::G_FNEARBYINT:
558 RTLIBCASE(NEARBYINT_F);
559 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
560 RTLIBCASE(ROUNDEVEN_F);
561 }
562 llvm_unreachable("Unknown libcall function");
563}
564
565/// True if an instruction is in tail position in its caller. Intended for
566/// legalizing libcalls as tail calls when possible.
568 const TargetInstrInfo &TII,
570 MachineBasicBlock &MBB = *MI.getParent();
571 const Function &F = MBB.getParent()->getFunction();
572
573 // Conservatively require the attributes of the call to match those of
574 // the return. Ignore NoAlias and NonNull because they don't affect the
575 // call sequence.
576 AttributeList CallerAttrs = F.getAttributes();
577 if (AttrBuilder(F.getContext(), CallerAttrs.getRetAttrs())
578 .removeAttribute(Attribute::NoAlias)
579 .removeAttribute(Attribute::NonNull)
580 .hasAttributes())
581 return false;
582
583 // It's not safe to eliminate the sign / zero extension of the return value.
584 if (CallerAttrs.hasRetAttr(Attribute::ZExt) ||
585 CallerAttrs.hasRetAttr(Attribute::SExt))
586 return false;
587
588 // Only tail call if the following instruction is a standard return or if we
589 // have a `thisreturn` callee, and a sequence like:
590 //
591 // G_MEMCPY %0, %1, %2
592 // $x0 = COPY %0
593 // RET_ReallyLR implicit $x0
594 auto Next = next_nodbg(MI.getIterator(), MBB.instr_end());
595 if (Next != MBB.instr_end() && Next->isCopy()) {
596 switch (MI.getOpcode()) {
597 default:
598 llvm_unreachable("unsupported opcode");
599 case TargetOpcode::G_BZERO:
600 return false;
601 case TargetOpcode::G_MEMCPY:
602 case TargetOpcode::G_MEMMOVE:
603 case TargetOpcode::G_MEMSET:
604 break;
605 }
606
607 Register VReg = MI.getOperand(0).getReg();
608 if (!VReg.isVirtual() || VReg != Next->getOperand(1).getReg())
609 return false;
610
611 Register PReg = Next->getOperand(0).getReg();
612 if (!PReg.isPhysical())
613 return false;
614
615 auto Ret = next_nodbg(Next, MBB.instr_end());
616 if (Ret == MBB.instr_end() || !Ret->isReturn())
617 return false;
618
619 if (Ret->getNumImplicitOperands() != 1)
620 return false;
621
622 if (PReg != Ret->getOperand(0).getReg())
623 return false;
624
625 // Skip over the COPY that we just validated.
626 Next = Ret;
627 }
628
629 if (Next == MBB.instr_end() || TII.isTailCall(*Next) || !Next->isReturn())
630 return false;
631
632 return true;
633}
634
637 const CallLowering::ArgInfo &Result,
639 const CallingConv::ID CC) {
640 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
641
643 Info.CallConv = CC;
645 Info.OrigRet = Result;
646 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
647 if (!CLI.lowerCall(MIRBuilder, Info))
649
651}
652
655 const CallLowering::ArgInfo &Result,
657 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
658 const char *Name = TLI.getLibcallName(Libcall);
659 const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall);
660 return createLibcall(MIRBuilder, Name, Result, Args, CC);
661}
662
663// Useful for libcalls where all operands have the same type.
666 Type *OpType) {
667 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
668
669 // FIXME: What does the original arg index mean here?
671 for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))
672 Args.push_back({MO.getReg(), OpType, 0});
673 return createLibcall(MIRBuilder, Libcall,
674 {MI.getOperand(0).getReg(), OpType, 0}, Args);
675}
676
679 MachineInstr &MI, LostDebugLocObserver &LocObserver) {
680 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
681
683 // Add all the args, except for the last which is an imm denoting 'tail'.
684 for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
685 Register Reg = MI.getOperand(i).getReg();
686
687 // Need derive an IR type for call lowering.
688 LLT OpLLT = MRI.getType(Reg);
689 Type *OpTy = nullptr;
690 if (OpLLT.isPointer())
691 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
692 else
693 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
694 Args.push_back({Reg, OpTy, 0});
695 }
696
697 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
698 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
699 RTLIB::Libcall RTLibcall;
700 unsigned Opc = MI.getOpcode();
701 switch (Opc) {
702 case TargetOpcode::G_BZERO:
703 RTLibcall = RTLIB::BZERO;
704 break;
705 case TargetOpcode::G_MEMCPY:
706 RTLibcall = RTLIB::MEMCPY;
707 Args[0].Flags[0].setReturned();
708 break;
709 case TargetOpcode::G_MEMMOVE:
710 RTLibcall = RTLIB::MEMMOVE;
711 Args[0].Flags[0].setReturned();
712 break;
713 case TargetOpcode::G_MEMSET:
714 RTLibcall = RTLIB::MEMSET;
715 Args[0].Flags[0].setReturned();
716 break;
717 default:
718 llvm_unreachable("unsupported opcode");
719 }
720 const char *Name = TLI.getLibcallName(RTLibcall);
721
722 // Unsupported libcall on the target.
723 if (!Name) {
724 LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
725 << MIRBuilder.getTII().getName(Opc) << "\n");
727 }
728
730 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
732 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0);
733 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() &&
734 isLibCallInTailPosition(MI, MIRBuilder.getTII(), MRI);
735
736 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
737 if (!CLI.lowerCall(MIRBuilder, Info))
739
740 if (Info.LoweredTailCall) {
741 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
742
743 // Check debug locations before removing the return.
744 LocObserver.checkpoint(true);
745
746 // We must have a return following the call (or debug insts) to get past
747 // isLibCallInTailPosition.
748 do {
749 MachineInstr *Next = MI.getNextNode();
750 assert(Next &&
751 (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) &&
752 "Expected instr following MI to be return or debug inst?");
753 // We lowered a tail call, so the call is now the return from the block.
754 // Delete the old return.
755 Next->eraseFromParent();
756 } while (MI.getNextNode());
757
758 // We expect to lose the debug location from the return.
759 LocObserver.checkpoint(false);
760 }
761
763}
764
765static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
766 Type *FromType) {
767 auto ToMVT = MVT::getVT(ToType);
768 auto FromMVT = MVT::getVT(FromType);
769
770 switch (Opcode) {
771 case TargetOpcode::G_FPEXT:
772 return RTLIB::getFPEXT(FromMVT, ToMVT);
773 case TargetOpcode::G_FPTRUNC:
774 return RTLIB::getFPROUND(FromMVT, ToMVT);
775 case TargetOpcode::G_FPTOSI:
776 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
777 case TargetOpcode::G_FPTOUI:
778 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
779 case TargetOpcode::G_SITOFP:
780 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
781 case TargetOpcode::G_UITOFP:
782 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
783 }
784 llvm_unreachable("Unsupported libcall function");
785}
786
789 Type *FromType) {
790 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
791 return createLibcall(MIRBuilder, Libcall,
792 {MI.getOperand(0).getReg(), ToType, 0},
793 {{MI.getOperand(1).getReg(), FromType, 0}});
794}
795
798 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
799 unsigned Size = LLTy.getSizeInBits();
800 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
801
802 switch (MI.getOpcode()) {
803 default:
804 return UnableToLegalize;
805 case TargetOpcode::G_MUL:
806 case TargetOpcode::G_SDIV:
807 case TargetOpcode::G_UDIV:
808 case TargetOpcode::G_SREM:
809 case TargetOpcode::G_UREM:
810 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
811 Type *HLTy = IntegerType::get(Ctx, Size);
812 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
813 if (Status != Legalized)
814 return Status;
815 break;
816 }
817 case TargetOpcode::G_FADD:
818 case TargetOpcode::G_FSUB:
819 case TargetOpcode::G_FMUL:
820 case TargetOpcode::G_FDIV:
821 case TargetOpcode::G_FMA:
822 case TargetOpcode::G_FPOW:
823 case TargetOpcode::G_FREM:
824 case TargetOpcode::G_FCOS:
825 case TargetOpcode::G_FSIN:
826 case TargetOpcode::G_FLOG10:
827 case TargetOpcode::G_FLOG:
828 case TargetOpcode::G_FLOG2:
829 case TargetOpcode::G_FEXP:
830 case TargetOpcode::G_FEXP2:
831 case TargetOpcode::G_FCEIL:
832 case TargetOpcode::G_FFLOOR:
833 case TargetOpcode::G_FMINNUM:
834 case TargetOpcode::G_FMAXNUM:
835 case TargetOpcode::G_FSQRT:
836 case TargetOpcode::G_FRINT:
837 case TargetOpcode::G_FNEARBYINT:
838 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
839 Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
840 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
841 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
842 return UnableToLegalize;
843 }
844 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
845 if (Status != Legalized)
846 return Status;
847 break;
848 }
849 case TargetOpcode::G_FPEXT:
850 case TargetOpcode::G_FPTRUNC: {
851 Type *FromTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg()));
852 Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg()));
853 if (!FromTy || !ToTy)
854 return UnableToLegalize;
856 if (Status != Legalized)
857 return Status;
858 break;
859 }
860 case TargetOpcode::G_FPTOSI:
861 case TargetOpcode::G_FPTOUI: {
862 // FIXME: Support other types
863 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
864 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
865 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
866 return UnableToLegalize;
868 MI, MIRBuilder,
869 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
870 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
871 if (Status != Legalized)
872 return Status;
873 break;
874 }
875 case TargetOpcode::G_SITOFP:
876 case TargetOpcode::G_UITOFP: {
877 // FIXME: Support other types
878 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
879 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
880 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
881 return UnableToLegalize;
883 MI, MIRBuilder,
884 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
885 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
886 if (Status != Legalized)
887 return Status;
888 break;
889 }
890 case TargetOpcode::G_BZERO:
891 case TargetOpcode::G_MEMCPY:
892 case TargetOpcode::G_MEMMOVE:
893 case TargetOpcode::G_MEMSET: {
894 LegalizeResult Result =
896 if (Result != Legalized)
897 return Result;
898 MI.eraseFromParent();
899 return Result;
900 }
901 }
902
903 MI.eraseFromParent();
904 return Legalized;
905}
906
908 unsigned TypeIdx,
909 LLT NarrowTy) {
910 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
911 uint64_t NarrowSize = NarrowTy.getSizeInBits();
912
913 switch (MI.getOpcode()) {
914 default:
915 return UnableToLegalize;
916 case TargetOpcode::G_IMPLICIT_DEF: {
917 Register DstReg = MI.getOperand(0).getReg();
918 LLT DstTy = MRI.getType(DstReg);
919
920 // If SizeOp0 is not an exact multiple of NarrowSize, emit
921 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
922 // FIXME: Although this would also be legal for the general case, it causes
923 // a lot of regressions in the emitted code (superfluous COPYs, artifact
924 // combines not being hit). This seems to be a problem related to the
925 // artifact combiner.
926 if (SizeOp0 % NarrowSize != 0) {
927 LLT ImplicitTy = NarrowTy;
928 if (DstTy.isVector())
929 ImplicitTy = LLT::vector(DstTy.getElementCount(), ImplicitTy);
930
931 Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0);
932 MIRBuilder.buildAnyExt(DstReg, ImplicitReg);
933
934 MI.eraseFromParent();
935 return Legalized;
936 }
937
938 int NumParts = SizeOp0 / NarrowSize;
939
941 for (int i = 0; i < NumParts; ++i)
942 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0));
943
944 if (DstTy.isVector())
945 MIRBuilder.buildBuildVector(DstReg, DstRegs);
946 else
947 MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs);
948 MI.eraseFromParent();
949 return Legalized;
950 }
951 case TargetOpcode::G_CONSTANT: {
952 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
953 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
954 unsigned TotalSize = Ty.getSizeInBits();
955 unsigned NarrowSize = NarrowTy.getSizeInBits();
956 int NumParts = TotalSize / NarrowSize;
957
959 for (int I = 0; I != NumParts; ++I) {
960 unsigned Offset = I * NarrowSize;
961 auto K = MIRBuilder.buildConstant(NarrowTy,
962 Val.lshr(Offset).trunc(NarrowSize));
963 PartRegs.push_back(K.getReg(0));
964 }
965
966 LLT LeftoverTy;
967 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
968 SmallVector<Register, 1> LeftoverRegs;
969 if (LeftoverBits != 0) {
970 LeftoverTy = LLT::scalar(LeftoverBits);
971 auto K = MIRBuilder.buildConstant(
972 LeftoverTy,
973 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
974 LeftoverRegs.push_back(K.getReg(0));
975 }
976
977 insertParts(MI.getOperand(0).getReg(),
978 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
979
980 MI.eraseFromParent();
981 return Legalized;
982 }
983 case TargetOpcode::G_SEXT:
984 case TargetOpcode::G_ZEXT:
985 case TargetOpcode::G_ANYEXT:
986 return narrowScalarExt(MI, TypeIdx, NarrowTy);
987 case TargetOpcode::G_TRUNC: {
988 if (TypeIdx != 1)
989 return UnableToLegalize;
990
991 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
992 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
993 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
994 return UnableToLegalize;
995 }
996
997 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
998 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0));
999 MI.eraseFromParent();
1000 return Legalized;
1001 }
1002
1003 case TargetOpcode::G_FREEZE: {
1004 if (TypeIdx != 0)
1005 return UnableToLegalize;
1006
1007 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1008 // Should widen scalar first
1009 if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0)
1010 return UnableToLegalize;
1011
1012 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg());
1014 for (unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1015 Parts.push_back(
1016 MIRBuilder.buildFreeze(NarrowTy, Unmerge.getReg(i)).getReg(0));
1017 }
1018
1019 MIRBuilder.buildMergeLikeInstr(MI.getOperand(0).getReg(), Parts);
1020 MI.eraseFromParent();
1021 return Legalized;
1022 }
1023 case TargetOpcode::G_ADD:
1024 case TargetOpcode::G_SUB:
1025 case TargetOpcode::G_SADDO:
1026 case TargetOpcode::G_SSUBO:
1027 case TargetOpcode::G_SADDE:
1028 case TargetOpcode::G_SSUBE:
1029 case TargetOpcode::G_UADDO:
1030 case TargetOpcode::G_USUBO:
1031 case TargetOpcode::G_UADDE:
1032 case TargetOpcode::G_USUBE:
1033 return narrowScalarAddSub(MI, TypeIdx, NarrowTy);
1034 case TargetOpcode::G_MUL:
1035 case TargetOpcode::G_UMULH:
1036 return narrowScalarMul(MI, NarrowTy);
1037 case TargetOpcode::G_EXTRACT:
1038 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
1039 case TargetOpcode::G_INSERT:
1040 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
1041 case TargetOpcode::G_LOAD: {
1042 auto &LoadMI = cast<GLoad>(MI);
1043 Register DstReg = LoadMI.getDstReg();
1044 LLT DstTy = MRI.getType(DstReg);
1045 if (DstTy.isVector())
1046 return UnableToLegalize;
1047
1048 if (8 * LoadMI.getMemSize() != DstTy.getSizeInBits()) {
1049 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
1050 MIRBuilder.buildLoad(TmpReg, LoadMI.getPointerReg(), LoadMI.getMMO());
1051 MIRBuilder.buildAnyExt(DstReg, TmpReg);
1052 LoadMI.eraseFromParent();
1053 return Legalized;
1054 }
1055
1056 return reduceLoadStoreWidth(LoadMI, TypeIdx, NarrowTy);
1057 }
1058 case TargetOpcode::G_ZEXTLOAD:
1059 case TargetOpcode::G_SEXTLOAD: {
1060 auto &LoadMI = cast<GExtLoad>(MI);
1061 Register DstReg = LoadMI.getDstReg();
1062 Register PtrReg = LoadMI.getPointerReg();
1063
1064 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
1065 auto &MMO = LoadMI.getMMO();
1066 unsigned MemSize = MMO.getSizeInBits();
1067
1068 if (MemSize == NarrowSize) {
1069 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
1070 } else if (MemSize < NarrowSize) {
1071 MIRBuilder.buildLoadInstr(LoadMI.getOpcode(), TmpReg, PtrReg, MMO);
1072 } else if (MemSize > NarrowSize) {
1073 // FIXME: Need to split the load.
1074 return UnableToLegalize;
1075 }
1076
1077 if (isa<GZExtLoad>(LoadMI))
1078 MIRBuilder.buildZExt(DstReg, TmpReg);
1079 else
1080 MIRBuilder.buildSExt(DstReg, TmpReg);
1081
1082 LoadMI.eraseFromParent();
1083 return Legalized;
1084 }
1085 case TargetOpcode::G_STORE: {
1086 auto &StoreMI = cast<GStore>(MI);
1087
1088 Register SrcReg = StoreMI.getValueReg();
1089 LLT SrcTy = MRI.getType(SrcReg);
1090 if (SrcTy.isVector())
1091 return UnableToLegalize;
1092
1093 int NumParts = SizeOp0 / NarrowSize;
1094 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
1095 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
1096 if (SrcTy.isVector() && LeftoverBits != 0)
1097 return UnableToLegalize;
1098
1099 if (8 * StoreMI.getMemSize() != SrcTy.getSizeInBits()) {
1100 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
1101 MIRBuilder.buildTrunc(TmpReg, SrcReg);
1102 MIRBuilder.buildStore(TmpReg, StoreMI.getPointerReg(), StoreMI.getMMO());
1103 StoreMI.eraseFromParent();
1104 return Legalized;
1105 }
1106
1107 return reduceLoadStoreWidth(StoreMI, 0, NarrowTy);
1108 }
1109 case TargetOpcode::G_SELECT:
1110 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
1111 case TargetOpcode::G_AND:
1112 case TargetOpcode::G_OR:
1113 case TargetOpcode::G_XOR: {
1114 // Legalize bitwise operation:
1115 // A = BinOp<Ty> B, C
1116 // into:
1117 // B1, ..., BN = G_UNMERGE_VALUES B
1118 // C1, ..., CN = G_UNMERGE_VALUES C
1119 // A1 = BinOp<Ty/N> B1, C2
1120 // ...
1121 // AN = BinOp<Ty/N> BN, CN
1122 // A = G_MERGE_VALUES A1, ..., AN
1123 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
1124 }
1125 case TargetOpcode::G_SHL:
1126 case TargetOpcode::G_LSHR:
1127 case TargetOpcode::G_ASHR:
1128 return narrowScalarShift(MI, TypeIdx, NarrowTy);
1129 case TargetOpcode::G_CTLZ:
1130 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1131 case TargetOpcode::G_CTTZ:
1132 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1133 case TargetOpcode::G_CTPOP:
1134 if (TypeIdx == 1)
1135 switch (MI.getOpcode()) {
1136 case TargetOpcode::G_CTLZ:
1137 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1138 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy);
1139 case TargetOpcode::G_CTTZ:
1140 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1141 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy);
1142 case TargetOpcode::G_CTPOP:
1143 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy);
1144 default:
1145 return UnableToLegalize;
1146 }
1147
1149 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1151 return Legalized;
1152 case TargetOpcode::G_INTTOPTR:
1153 if (TypeIdx != 1)
1154 return UnableToLegalize;
1155
1157 narrowScalarSrc(MI, NarrowTy, 1);
1159 return Legalized;
1160 case TargetOpcode::G_PTRTOINT:
1161 if (TypeIdx != 0)
1162 return UnableToLegalize;
1163
1165 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1167 return Legalized;
1168 case TargetOpcode::G_PHI: {
1169 // FIXME: add support for when SizeOp0 isn't an exact multiple of
1170 // NarrowSize.
1171 if (SizeOp0 % NarrowSize != 0)
1172 return UnableToLegalize;
1173
1174 unsigned NumParts = SizeOp0 / NarrowSize;
1175 SmallVector<Register, 2> DstRegs(NumParts);
1176 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1178 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1179 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
1181 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
1182 SrcRegs[i / 2]);
1183 }
1184 MachineBasicBlock &MBB = *MI.getParent();
1186 for (unsigned i = 0; i < NumParts; ++i) {
1187 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
1189 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
1190 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1191 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
1192 }
1194 MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), DstRegs);
1196 MI.eraseFromParent();
1197 return Legalized;
1198 }
1199 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1200 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1201 if (TypeIdx != 2)
1202 return UnableToLegalize;
1203
1204 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1206 narrowScalarSrc(MI, NarrowTy, OpIdx);
1208 return Legalized;
1209 }
1210 case TargetOpcode::G_ICMP: {
1211 Register LHS = MI.getOperand(2).getReg();
1212 LLT SrcTy = MRI.getType(LHS);
1213 uint64_t SrcSize = SrcTy.getSizeInBits();
1214 CmpInst::Predicate Pred =
1215 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
1216
1217 // TODO: Handle the non-equality case for weird sizes.
1218 if (NarrowSize * 2 != SrcSize && !ICmpInst::isEquality(Pred))
1219 return UnableToLegalize;
1220
1221 LLT LeftoverTy; // Example: s88 -> s64 (NarrowTy) + s24 (leftover)
1222 SmallVector<Register, 4> LHSPartRegs, LHSLeftoverRegs;
1223 if (!extractParts(LHS, SrcTy, NarrowTy, LeftoverTy, LHSPartRegs,
1224 LHSLeftoverRegs))
1225 return UnableToLegalize;
1226
1227 LLT Unused; // Matches LeftoverTy; G_ICMP LHS and RHS are the same type.
1228 SmallVector<Register, 4> RHSPartRegs, RHSLeftoverRegs;
1229 if (!extractParts(MI.getOperand(3).getReg(), SrcTy, NarrowTy, Unused,
1230 RHSPartRegs, RHSLeftoverRegs))
1231 return UnableToLegalize;
1232
1233 // We now have the LHS and RHS of the compare split into narrow-type
1234 // registers, plus potentially some leftover type.
1235 Register Dst = MI.getOperand(0).getReg();
1236 LLT ResTy = MRI.getType(Dst);
1237 if (ICmpInst::isEquality(Pred)) {
1238 // For each part on the LHS and RHS, keep track of the result of XOR-ing
1239 // them together. For each equal part, the result should be all 0s. For
1240 // each non-equal part, we'll get at least one 1.
1241 auto Zero = MIRBuilder.buildConstant(NarrowTy, 0);
1243 for (auto LHSAndRHS : zip(LHSPartRegs, RHSPartRegs)) {
1244 auto LHS = std::get<0>(LHSAndRHS);
1245 auto RHS = std::get<1>(LHSAndRHS);
1246 auto Xor = MIRBuilder.buildXor(NarrowTy, LHS, RHS).getReg(0);
1247 Xors.push_back(Xor);
1248 }
1249
1250 // Build a G_XOR for each leftover register. Each G_XOR must be widened
1251 // to the desired narrow type so that we can OR them together later.
1252 SmallVector<Register, 4> WidenedXors;
1253 for (auto LHSAndRHS : zip(LHSLeftoverRegs, RHSLeftoverRegs)) {
1254 auto LHS = std::get<0>(LHSAndRHS);
1255 auto RHS = std::get<1>(LHSAndRHS);
1256 auto Xor = MIRBuilder.buildXor(LeftoverTy, LHS, RHS).getReg(0);
1257 LLT GCDTy = extractGCDType(WidenedXors, NarrowTy, LeftoverTy, Xor);
1258 buildLCMMergePieces(LeftoverTy, NarrowTy, GCDTy, WidenedXors,
1259 /* PadStrategy = */ TargetOpcode::G_ZEXT);
1260 Xors.insert(Xors.end(), WidenedXors.begin(), WidenedXors.end());
1261 }
1262
1263 // Now, for each part we broke up, we know if they are equal/not equal
1264 // based off the G_XOR. We can OR these all together and compare against
1265 // 0 to get the result.
1266 assert(Xors.size() >= 2 && "Should have gotten at least two Xors?");
1267 auto Or = MIRBuilder.buildOr(NarrowTy, Xors[0], Xors[1]);
1268 for (unsigned I = 2, E = Xors.size(); I < E; ++I)
1269 Or = MIRBuilder.buildOr(NarrowTy, Or, Xors[I]);
1270 MIRBuilder.buildICmp(Pred, Dst, Or, Zero);
1271 } else {
1272 // TODO: Handle non-power-of-two types.
1273 assert(LHSPartRegs.size() == 2 && "Expected exactly 2 LHS part regs?");
1274 assert(RHSPartRegs.size() == 2 && "Expected exactly 2 RHS part regs?");
1275 Register LHSL = LHSPartRegs[0];
1276 Register LHSH = LHSPartRegs[1];
1277 Register RHSL = RHSPartRegs[0];
1278 Register RHSH = RHSPartRegs[1];
1279 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH);
1280 MachineInstrBuilder CmpHEQ =
1283 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL);
1284 MIRBuilder.buildSelect(Dst, CmpHEQ, CmpLU, CmpH);
1285 }
1286 MI.eraseFromParent();
1287 return Legalized;
1288 }
1289 case TargetOpcode::G_SEXT_INREG: {
1290 if (TypeIdx != 0)
1291 return UnableToLegalize;
1292
1293 int64_t SizeInBits = MI.getOperand(2).getImm();
1294
1295 // So long as the new type has more bits than the bits we're extending we
1296 // don't need to break it apart.
1297 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) {
1299 // We don't lose any non-extension bits by truncating the src and
1300 // sign-extending the dst.
1301 MachineOperand &MO1 = MI.getOperand(1);
1302 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1);
1303 MO1.setReg(TruncMIB.getReg(0));
1304
1305 MachineOperand &MO2 = MI.getOperand(0);
1306 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy);
1308 MIRBuilder.buildSExt(MO2, DstExt);
1309 MO2.setReg(DstExt);
1311 return Legalized;
1312 }
1313
1314 // Break it apart. Components below the extension point are unmodified. The
1315 // component containing the extension point becomes a narrower SEXT_INREG.
1316 // Components above it are ashr'd from the component containing the
1317 // extension point.
1318 if (SizeOp0 % NarrowSize != 0)
1319 return UnableToLegalize;
1320 int NumParts = SizeOp0 / NarrowSize;
1321
1322 // List the registers where the destination will be scattered.
1324 // List the registers where the source will be split.
1326
1327 // Create all the temporary registers.
1328 for (int i = 0; i < NumParts; ++i) {
1329 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
1330
1331 SrcRegs.push_back(SrcReg);
1332 }
1333
1334 // Explode the big arguments into smaller chunks.
1335 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1));
1336
1337 Register AshrCstReg =
1338 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1)
1339 .getReg(0);
1340 Register FullExtensionReg = 0;
1341 Register PartialExtensionReg = 0;
1342
1343 // Do the operation on each small part.
1344 for (int i = 0; i < NumParts; ++i) {
1345 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits)
1346 DstRegs.push_back(SrcRegs[i]);
1347 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) {
1348 assert(PartialExtensionReg &&
1349 "Expected to visit partial extension before full");
1350 if (FullExtensionReg) {
1351 DstRegs.push_back(FullExtensionReg);
1352 continue;
1353 }
1354 DstRegs.push_back(
1355 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg)
1356 .getReg(0));
1357 FullExtensionReg = DstRegs.back();
1358 } else {
1359 DstRegs.push_back(
1361 .buildInstr(
1362 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1363 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
1364 .getReg(0));
1365 PartialExtensionReg = DstRegs.back();
1366 }
1367 }
1368
1369 // Gather the destination registers into the final destination.
1370 Register DstReg = MI.getOperand(0).getReg();
1371 MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs);
1372 MI.eraseFromParent();
1373 return Legalized;
1374 }
1375 case TargetOpcode::G_BSWAP:
1376 case TargetOpcode::G_BITREVERSE: {
1377 if (SizeOp0 % NarrowSize != 0)
1378 return UnableToLegalize;
1379
1381 SmallVector<Register, 2> SrcRegs, DstRegs;
1382 unsigned NumParts = SizeOp0 / NarrowSize;
1383 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
1384
1385 for (unsigned i = 0; i < NumParts; ++i) {
1386 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
1387 {SrcRegs[NumParts - 1 - i]});
1388 DstRegs.push_back(DstPart.getReg(0));
1389 }
1390
1391 MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), DstRegs);
1392
1394 MI.eraseFromParent();
1395 return Legalized;
1396 }
1397 case TargetOpcode::G_PTR_ADD:
1398 case TargetOpcode::G_PTRMASK: {
1399 if (TypeIdx != 1)
1400 return UnableToLegalize;
1402 narrowScalarSrc(MI, NarrowTy, 2);
1404 return Legalized;
1405 }
1406 case TargetOpcode::G_FPTOUI:
1407 case TargetOpcode::G_FPTOSI:
1408 return narrowScalarFPTOI(MI, TypeIdx, NarrowTy);
1409 case TargetOpcode::G_FPEXT:
1410 if (TypeIdx != 0)
1411 return UnableToLegalize;
1413 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT);
1415 return Legalized;
1416 }
1417}
1418
1420 LLT Ty = MRI.getType(Val);
1421 if (Ty.isScalar())
1422 return Val;
1423
1425 LLT NewTy = LLT::scalar(Ty.getSizeInBits());
1426 if (Ty.isPointer()) {
1427 if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace()))
1428 return Register();
1429 return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0);
1430 }
1431
1432 Register NewVal = Val;
1433
1434 assert(Ty.isVector());
1435 LLT EltTy = Ty.getElementType();
1436 if (EltTy.isPointer())
1437 NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0);
1438 return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0);
1439}
1440
1442 unsigned OpIdx, unsigned ExtOpcode) {
1443 MachineOperand &MO = MI.getOperand(OpIdx);
1444 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO});
1445 MO.setReg(ExtB.getReg(0));
1446}
1447
1449 unsigned OpIdx) {
1450 MachineOperand &MO = MI.getOperand(OpIdx);
1451 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO);
1452 MO.setReg(ExtB.getReg(0));
1453}
1454
1456 unsigned OpIdx, unsigned TruncOpcode) {
1457 MachineOperand &MO = MI.getOperand(OpIdx);
1458 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1460 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt});
1461 MO.setReg(DstExt);
1462}
1463
1465 unsigned OpIdx, unsigned ExtOpcode) {
1466 MachineOperand &MO = MI.getOperand(OpIdx);
1467 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
1469 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc});
1470 MO.setReg(DstTrunc);
1471}
1472
1474 unsigned OpIdx) {
1475 MachineOperand &MO = MI.getOperand(OpIdx);
1477 Register Dst = MO.getReg();
1478 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1479 MO.setReg(DstExt);
1481}
1482
1484 unsigned OpIdx) {
1485 MachineOperand &MO = MI.getOperand(OpIdx);
1488}
1489
1490void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
1491 MachineOperand &Op = MI.getOperand(OpIdx);
1492 Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0));
1493}
1494
1495void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
1496 MachineOperand &MO = MI.getOperand(OpIdx);
1497 Register CastDst = MRI.createGenericVirtualRegister(CastTy);
1499 MIRBuilder.buildBitcast(MO, CastDst);
1500 MO.setReg(CastDst);
1501}
1502
1504LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
1505 LLT WideTy) {
1506 if (TypeIdx != 1)
1507 return UnableToLegalize;
1508
1509 Register DstReg = MI.getOperand(0).getReg();
1510 LLT DstTy = MRI.getType(DstReg);
1511 if (DstTy.isVector())
1512 return UnableToLegalize;
1513
1514 Register Src1 = MI.getOperand(1).getReg();
1515 LLT SrcTy = MRI.getType(Src1);
1516 const int DstSize = DstTy.getSizeInBits();
1517 const int SrcSize = SrcTy.getSizeInBits();
1518 const int WideSize = WideTy.getSizeInBits();
1519 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1520
1521 unsigned NumOps = MI.getNumOperands();
1522 unsigned NumSrc = MI.getNumOperands() - 1;
1523 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1524
1525 if (WideSize >= DstSize) {
1526 // Directly pack the bits in the target type.
1527 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0);
1528
1529 for (unsigned I = 2; I != NumOps; ++I) {
1530 const unsigned Offset = (I - 1) * PartSize;
1531
1532 Register SrcReg = MI.getOperand(I).getReg();
1533 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
1534
1535 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
1536
1537 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
1538 MRI.createGenericVirtualRegister(WideTy);
1539
1540 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
1541 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
1542 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
1543 ResultReg = NextResult;
1544 }
1545
1546 if (WideSize > DstSize)
1547 MIRBuilder.buildTrunc(DstReg, ResultReg);
1548 else if (DstTy.isPointer())
1549 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
1550
1551 MI.eraseFromParent();
1552 return Legalized;
1553 }
1554
1555 // Unmerge the original values to the GCD type, and recombine to the next
1556 // multiple greater than the original type.
1557 //
1558 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
1559 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
1560 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
1561 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
1562 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
1563 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
1564 // %12:_(s12) = G_MERGE_VALUES %10, %11
1565 //
1566 // Padding with undef if necessary:
1567 //
1568 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
1569 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
1570 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
1571 // %7:_(s2) = G_IMPLICIT_DEF
1572 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
1573 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
1574 // %10:_(s12) = G_MERGE_VALUES %8, %9
1575
1576 const int GCD = std::gcd(SrcSize, WideSize);
1577 LLT GCDTy = LLT::scalar(GCD);
1578
1580 SmallVector<Register, 8> NewMergeRegs;
1581 SmallVector<Register, 8> Unmerges;
1582 LLT WideDstTy = LLT::scalar(NumMerge * WideSize);
1583
1584 // Decompose the original operands if they don't evenly divide.
1585 for (const MachineOperand &MO : llvm::drop_begin(MI.operands())) {
1586 Register SrcReg = MO.getReg();
1587 if (GCD == SrcSize) {
1588 Unmerges.push_back(SrcReg);
1589 } else {
1590 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
1591 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1592 Unmerges.push_back(Unmerge.getReg(J));
1593 }
1594 }
1595
1596 // Pad with undef to the next size that is a multiple of the requested size.
1597 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
1598 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
1599 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
1600 Unmerges.push_back(UndefReg);
1601 }
1602
1603 const int PartsPerGCD = WideSize / GCD;
1604
1605 // Build merges of each piece.
1606 ArrayRef<Register> Slicer(Unmerges);
1607 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1608 auto Merge =
1609 MIRBuilder.buildMergeLikeInstr(WideTy, Slicer.take_front(PartsPerGCD));
1610 NewMergeRegs.push_back(Merge.getReg(0));
1611 }
1612
1613 // A truncate may be necessary if the requested type doesn't evenly divide the
1614 // original result type.
1615 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
1616 MIRBuilder.buildMergeLikeInstr(DstReg, NewMergeRegs);
1617 } else {
1618 auto FinalMerge = MIRBuilder.buildMergeLikeInstr(WideDstTy, NewMergeRegs);
1619 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0));
1620 }
1621
1622 MI.eraseFromParent();
1623 return Legalized;
1624}
1625
1627LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
1628 LLT WideTy) {
1629 if (TypeIdx != 0)
1630 return UnableToLegalize;
1631
1632 int NumDst = MI.getNumOperands() - 1;
1633 Register SrcReg = MI.getOperand(NumDst).getReg();
1634 LLT SrcTy = MRI.getType(SrcReg);
1635 if (SrcTy.isVector())
1636 return UnableToLegalize;
1637
1638 Register Dst0Reg = MI.getOperand(0).getReg();
1639 LLT DstTy = MRI.getType(Dst0Reg);
1640 if (!DstTy.isScalar())
1641 return UnableToLegalize;
1642
1643 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
1644 if (SrcTy.isPointer()) {
1646 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) {
1647 LLVM_DEBUG(
1648 dbgs() << "Not casting non-integral address space integer\n");
1649 return UnableToLegalize;
1650 }
1651
1652 SrcTy = LLT::scalar(SrcTy.getSizeInBits());
1653 SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0);
1654 }
1655
1656 // Widen SrcTy to WideTy. This does not affect the result, but since the
1657 // user requested this size, it is probably better handled than SrcTy and
1658 // should reduce the total number of legalization artifacts.
1659 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1660 SrcTy = WideTy;
1661 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0);
1662 }
1663
1664 // Theres no unmerge type to target. Directly extract the bits from the
1665 // source type
1666 unsigned DstSize = DstTy.getSizeInBits();
1667
1668 MIRBuilder.buildTrunc(Dst0Reg, SrcReg);
1669 for (int I = 1; I != NumDst; ++I) {
1670 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I);
1671 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt);
1672 MIRBuilder.buildTrunc(MI.getOperand(I), Shr);
1673 }
1674
1675 MI.eraseFromParent();
1676 return Legalized;
1677 }
1678
1679 // Extend the source to a wider type.
1680 LLT LCMTy = getLCMType(SrcTy, WideTy);
1681
1682 Register WideSrc = SrcReg;
1683 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
1684 // TODO: If this is an integral address space, cast to integer and anyext.
1685 if (SrcTy.isPointer()) {
1686 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n");
1687 return UnableToLegalize;
1688 }
1689
1690 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0);
1691 }
1692
1693 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc);
1694
1695 // Create a sequence of unmerges and merges to the original results. Since we
1696 // may have widened the source, we will need to pad the results with dead defs
1697 // to cover the source register.
1698 // e.g. widen s48 to s64:
1699 // %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96)
1700 //
1701 // =>
1702 // %4:_(s192) = G_ANYEXT %0:_(s96)
1703 // %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge
1704 // ; unpack to GCD type, with extra dead defs
1705 // %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64)
1706 // %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64)
1707 // dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64)
1708 // %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination
1709 // %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination
1710 const LLT GCDTy = getGCDType(WideTy, DstTy);
1711 const int NumUnmerge = Unmerge->getNumOperands() - 1;
1712 const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits();
1713
1714 // Directly unmerge to the destination without going through a GCD type
1715 // if possible
1716 if (PartsPerRemerge == 1) {
1717 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
1718
1719 for (int I = 0; I != NumUnmerge; ++I) {
1720 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
1721
1722 for (int J = 0; J != PartsPerUnmerge; ++J) {
1723 int Idx = I * PartsPerUnmerge + J;
1724 if (Idx < NumDst)
1725 MIB.addDef(MI.getOperand(Idx).getReg());
1726 else {
1727 // Create dead def for excess components.
1728 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
1729 }
1730 }
1731
1732 MIB.addUse(Unmerge.getReg(I));
1733 }
1734 } else {
1736 for (int J = 0; J != NumUnmerge; ++J)
1737 extractGCDType(Parts, GCDTy, Unmerge.getReg(J));
1738
1739 SmallVector<Register, 8> RemergeParts;
1740 for (int I = 0; I != NumDst; ++I) {
1741 for (int J = 0; J < PartsPerRemerge; ++J) {
1742 const int Idx = I * PartsPerRemerge + J;
1743 RemergeParts.emplace_back(Parts[Idx]);
1744 }
1745
1746 MIRBuilder.buildMergeLikeInstr(MI.getOperand(I).getReg(), RemergeParts);
1747 RemergeParts.clear();
1748 }
1749 }
1750
1751 MI.eraseFromParent();
1752 return Legalized;
1753}
1754
1756LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
1757 LLT WideTy) {
1758 Register DstReg = MI.getOperand(0).getReg();
1759 Register SrcReg = MI.getOperand(1).getReg();
1760 LLT SrcTy = MRI.getType(SrcReg);
1761
1762 LLT DstTy = MRI.getType(DstReg);
1763 unsigned Offset = MI.getOperand(2).getImm();
1764
1765 if (TypeIdx == 0) {
1766 if (SrcTy.isVector() || DstTy.isVector())
1767 return UnableToLegalize;
1768
1769 SrcOp Src(SrcReg);
1770 if (SrcTy.isPointer()) {
1771 // Extracts from pointers can be handled only if they are really just
1772 // simple integers.
1774 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
1775 return UnableToLegalize;
1776
1777 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
1778 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
1779 SrcTy = SrcAsIntTy;
1780 }
1781
1782 if (DstTy.isPointer())
1783 return UnableToLegalize;
1784
1785 if (Offset == 0) {
1786 // Avoid a shift in the degenerate case.
1787 MIRBuilder.buildTrunc(DstReg,
1788 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
1789 MI.eraseFromParent();
1790 return Legalized;
1791 }
1792
1793 // Do a shift in the source type.
1794 LLT ShiftTy = SrcTy;
1795 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1796 Src = MIRBuilder.buildAnyExt(WideTy, Src);
1797 ShiftTy = WideTy;
1798 }
1799
1800 auto LShr = MIRBuilder.buildLShr(
1801 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
1802 MIRBuilder.buildTrunc(DstReg, LShr);
1803 MI.eraseFromParent();
1804 return Legalized;
1805 }
1806
1807 if (SrcTy.isScalar()) {
1809 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1811 return Legalized;
1812 }
1813
1814 if (!SrcTy.isVector())
1815 return UnableToLegalize;
1816
1817 if (DstTy != SrcTy.getElementType())
1818 return UnableToLegalize;
1819
1820 if (Offset % SrcTy.getScalarSizeInBits() != 0)
1821 return UnableToLegalize;
1822
1824 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1825
1826 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1827 Offset);
1828 widenScalarDst(MI, WideTy.getScalarType(), 0);
1830 return Legalized;
1831}
1832
1834LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1835 LLT WideTy) {
1836 if (TypeIdx != 0 || WideTy.isVector())
1837 return UnableToLegalize;
1839 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1840 widenScalarDst(MI, WideTy);
1842 return Legalized;
1843}
1844
1846LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx,
1847 LLT WideTy) {
1848 unsigned Opcode;
1849 unsigned ExtOpcode;
1850 std::optional<Register> CarryIn;
1851 switch (MI.getOpcode()) {
1852 default:
1853 llvm_unreachable("Unexpected opcode!");
1854 case TargetOpcode::G_SADDO:
1855 Opcode = TargetOpcode::G_ADD;
1856 ExtOpcode = TargetOpcode::G_SEXT;
1857 break;
1858 case TargetOpcode::G_SSUBO:
1859 Opcode = TargetOpcode::G_SUB;
1860 ExtOpcode = TargetOpcode::G_SEXT;
1861 break;
1862 case TargetOpcode::G_UADDO:
1863 Opcode = TargetOpcode::G_ADD;
1864 ExtOpcode = TargetOpcode::G_ZEXT;
1865 break;
1866 case TargetOpcode::G_USUBO:
1867 Opcode = TargetOpcode::G_SUB;
1868 ExtOpcode = TargetOpcode::G_ZEXT;
1869 break;
1870 case TargetOpcode::G_SADDE:
1871 Opcode = TargetOpcode::G_UADDE;
1872 ExtOpcode = TargetOpcode::G_SEXT;
1873 CarryIn = MI.getOperand(4).getReg();
1874 break;
1875 case TargetOpcode::G_SSUBE:
1876 Opcode = TargetOpcode::G_USUBE;
1877 ExtOpcode = TargetOpcode::G_SEXT;
1878 CarryIn = MI.getOperand(4).getReg();
1879 break;
1880 case TargetOpcode::G_UADDE:
1881 Opcode = TargetOpcode::G_UADDE;
1882 ExtOpcode = TargetOpcode::G_ZEXT;
1883 CarryIn = MI.getOperand(4).getReg();
1884 break;
1885 case TargetOpcode::G_USUBE:
1886 Opcode = TargetOpcode::G_USUBE;
1887 ExtOpcode = TargetOpcode::G_ZEXT;
1888 CarryIn = MI.getOperand(4).getReg();
1889 break;
1890 }
1891
1892 if (TypeIdx == 1) {
1893 unsigned BoolExtOp = MIRBuilder.getBoolExtOp(WideTy.isVector(), false);
1894
1896 if (CarryIn)
1897 widenScalarSrc(MI, WideTy, 4, BoolExtOp);
1898 widenScalarDst(MI, WideTy, 1);
1899
1901 return Legalized;
1902 }
1903
1904 auto LHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(2)});
1905 auto RHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(3)});
1906 // Do the arithmetic in the larger type.
1907 Register NewOp;
1908 if (CarryIn) {
1909 LLT CarryOutTy = MRI.getType(MI.getOperand(1).getReg());
1910 NewOp = MIRBuilder
1911 .buildInstr(Opcode, {WideTy, CarryOutTy},
1912 {LHSExt, RHSExt, *CarryIn})
1913 .getReg(0);
1914 } else {
1915 NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSExt, RHSExt}).getReg(0);
1916 }
1917 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1918 auto TruncOp = MIRBuilder.buildTrunc(OrigTy, NewOp);
1919 auto ExtOp = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {TruncOp});
1920 // There is no overflow if the ExtOp is the same as NewOp.
1921 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, ExtOp);
1922 // Now trunc the NewOp to the original result.
1923 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp);
1924 MI.eraseFromParent();
1925 return Legalized;
1926}
1927
1929LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
1930 LLT WideTy) {
1931 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT ||
1932 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
1933 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
1934 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
1935 MI.getOpcode() == TargetOpcode::G_USHLSAT;
1936 // We can convert this to:
1937 // 1. Any extend iN to iM
1938 // 2. SHL by M-N
1939 // 3. [US][ADD|SUB|SHL]SAT
1940 // 4. L/ASHR by M-N
1941 //
1942 // It may be more efficient to lower this to a min and a max operation in
1943 // the higher precision arithmetic if the promoted operation isn't legal,
1944 // but this decision is up to the target's lowering request.
1945 Register DstReg = MI.getOperand(0).getReg();
1946
1947 unsigned NewBits = WideTy.getScalarSizeInBits();
1948 unsigned SHLAmount = NewBits - MRI.getType(DstReg).getScalarSizeInBits();
1949
1950 // Shifts must zero-extend the RHS to preserve the unsigned quantity, and
1951 // must not left shift the RHS to preserve the shift amount.
1952 auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1));
1953 auto RHS = IsShift ? MIRBuilder.buildZExt(WideTy, MI.getOperand(2))
1954 : MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2));
1955 auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount);
1956 auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK);
1957 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(WideTy, RHS, ShiftK);
1958
1959 auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy},
1960 {ShiftL, ShiftR}, MI.getFlags());
1961
1962 // Use a shift that will preserve the number of sign bits when the trunc is
1963 // folded away.
1964 auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK)
1965 : MIRBuilder.buildLShr(WideTy, WideInst, ShiftK);
1966
1967 MIRBuilder.buildTrunc(DstReg, Result);
1968 MI.eraseFromParent();
1969 return Legalized;
1970}
1971
1973LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
1974 LLT WideTy) {
1975 if (TypeIdx == 1) {
1977 widenScalarDst(MI, WideTy, 1);
1979 return Legalized;
1980 }
1981
1982 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO;
1983 Register Result = MI.getOperand(0).getReg();
1984 Register OriginalOverflow = MI.getOperand(1).getReg();
1985 Register LHS = MI.getOperand(2).getReg();
1986 Register RHS = MI.getOperand(3).getReg();
1987 LLT SrcTy = MRI.getType(LHS);
1988 LLT OverflowTy = MRI.getType(OriginalOverflow);
1989 unsigned SrcBitWidth = SrcTy.getScalarSizeInBits();
1990
1991 // To determine if the result overflowed in the larger type, we extend the
1992 // input to the larger type, do the multiply (checking if it overflows),
1993 // then also check the high bits of the result to see if overflow happened
1994 // there.
1995 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1996 auto LeftOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {LHS});
1997 auto RightOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {RHS});
1998
1999 auto Mulo = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy, OverflowTy},
2000 {LeftOperand, RightOperand});
2001 auto Mul = Mulo->getOperand(0);
2002 MIRBuilder.buildTrunc(Result, Mul);
2003
2004 MachineInstrBuilder ExtResult;
2005 // Overflow occurred if it occurred in the larger type, or if the high part
2006 // of the result does not zero/sign-extend the low part. Check this second
2007 // possibility first.
2008 if (IsSigned) {
2009 // For signed, overflow occurred when the high part does not sign-extend
2010 // the low part.
2011 ExtResult = MIRBuilder.buildSExtInReg(WideTy, Mul, SrcBitWidth);
2012 } else {
2013 // Unsigned overflow occurred when the high part does not zero-extend the
2014 // low part.
2015 ExtResult = MIRBuilder.buildZExtInReg(WideTy, Mul, SrcBitWidth);
2016 }
2017
2018 // Multiplication cannot overflow if the WideTy is >= 2 * original width,
2019 // so we don't need to check the overflow result of larger type Mulo.
2020 if (WideTy.getScalarSizeInBits() < 2 * SrcBitWidth) {
2021 auto Overflow =
2022 MIRBuilder.buildICmp(CmpInst::ICMP_NE, OverflowTy, Mul, ExtResult);
2023 // Finally check if the multiplication in the larger type itself overflowed.
2024 MIRBuilder.buildOr(OriginalOverflow, Mulo->getOperand(1), Overflow);
2025 } else {
2026 MIRBuilder.buildICmp(CmpInst::ICMP_NE, OriginalOverflow, Mul, ExtResult);
2027 }
2028 MI.eraseFromParent();
2029 return Legalized;
2030}
2031
2034 switch (MI.getOpcode()) {
2035 default:
2036 return UnableToLegalize;
2037 case TargetOpcode::G_ATOMICRMW_XCHG:
2038 case TargetOpcode::G_ATOMICRMW_ADD:
2039 case TargetOpcode::G_ATOMICRMW_SUB:
2040 case TargetOpcode::G_ATOMICRMW_AND:
2041 case TargetOpcode::G_ATOMICRMW_OR:
2042 case TargetOpcode::G_ATOMICRMW_XOR:
2043 case TargetOpcode::G_ATOMICRMW_MIN:
2044 case TargetOpcode::G_ATOMICRMW_MAX:
2045 case TargetOpcode::G_ATOMICRMW_UMIN:
2046 case TargetOpcode::G_ATOMICRMW_UMAX:
2047 assert(TypeIdx == 0 && "atomicrmw with second scalar type");
2049 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2050 widenScalarDst(MI, WideTy, 0);
2052 return Legalized;
2053 case TargetOpcode::G_ATOMIC_CMPXCHG:
2054 assert(TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type");
2056 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2057 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
2058 widenScalarDst(MI, WideTy, 0);
2060 return Legalized;
2061 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2062 if (TypeIdx == 0) {
2064 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
2065 widenScalarSrc(MI, WideTy, 4, TargetOpcode::G_ANYEXT);
2066 widenScalarDst(MI, WideTy, 0);
2068 return Legalized;
2069 }
2070 assert(TypeIdx == 1 &&
2071 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2073 widenScalarDst(MI, WideTy, 1);
2075 return Legalized;
2076 case TargetOpcode::G_EXTRACT:
2077 return widenScalarExtract(MI, TypeIdx, WideTy);
2078 case TargetOpcode::G_INSERT:
2079 return widenScalarInsert(MI, TypeIdx, WideTy);
2080 case TargetOpcode::G_MERGE_VALUES:
2081 return widenScalarMergeValues(MI, TypeIdx, WideTy);
2082 case TargetOpcode::G_UNMERGE_VALUES:
2083 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
2084 case TargetOpcode::G_SADDO:
2085 case TargetOpcode::G_SSUBO:
2086 case TargetOpcode::G_UADDO:
2087 case TargetOpcode::G_USUBO:
2088 case TargetOpcode::G_SADDE:
2089 case TargetOpcode::G_SSUBE:
2090 case TargetOpcode::G_UADDE:
2091 case TargetOpcode::G_USUBE:
2092 return widenScalarAddSubOverflow(MI, TypeIdx, WideTy);
2093 case TargetOpcode::G_UMULO:
2094 case TargetOpcode::G_SMULO:
2095 return widenScalarMulo(MI, TypeIdx, WideTy);
2096 case TargetOpcode::G_SADDSAT:
2097 case TargetOpcode::G_SSUBSAT:
2098 case TargetOpcode::G_SSHLSAT:
2099 case TargetOpcode::G_UADDSAT:
2100 case TargetOpcode::G_USUBSAT:
2101 case TargetOpcode::G_USHLSAT:
2102 return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
2103 case TargetOpcode::G_CTTZ:
2104 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2105 case TargetOpcode::G_CTLZ:
2106 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2107 case TargetOpcode::G_CTPOP: {
2108 if (TypeIdx == 0) {
2110 widenScalarDst(MI, WideTy, 0);
2112 return Legalized;
2113 }
2114
2115 Register SrcReg = MI.getOperand(1).getReg();
2116
2117 // First extend the input.
2118 unsigned ExtOpc = MI.getOpcode() == TargetOpcode::G_CTTZ ||
2119 MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF
2120 ? TargetOpcode::G_ANYEXT
2121 : TargetOpcode::G_ZEXT;
2122 auto MIBSrc = MIRBuilder.buildInstr(ExtOpc, {WideTy}, {SrcReg});
2123 LLT CurTy = MRI.getType(SrcReg);
2124 unsigned NewOpc = MI.getOpcode();
2125 if (NewOpc == TargetOpcode::G_CTTZ) {
2126 // The count is the same in the larger type except if the original
2127 // value was zero. This can be handled by setting the bit just off
2128 // the top of the original type.
2129 auto TopBit =
2131 MIBSrc = MIRBuilder.buildOr(
2132 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
2133 // Now we know the operand is non-zero, use the more relaxed opcode.
2134 NewOpc = TargetOpcode::G_CTTZ_ZERO_UNDEF;
2135 }
2136
2137 // Perform the operation at the larger size.
2138 auto MIBNewOp = MIRBuilder.buildInstr(NewOpc, {WideTy}, {MIBSrc});
2139 // This is already the correct result for CTPOP and CTTZs
2140 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
2141 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
2142 // The correct result is NewOp - (Difference in widety and current ty).
2143 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
2144 MIBNewOp = MIRBuilder.buildSub(
2145 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff));
2146 }
2147
2148 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
2149 MI.eraseFromParent();
2150 return Legalized;
2151 }
2152 case TargetOpcode::G_BSWAP: {
2154 Register DstReg = MI.getOperand(0).getReg();
2155
2156 Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
2157 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
2158 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
2159 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2160
2161 MI.getOperand(0).setReg(DstExt);
2162
2164
2165 LLT Ty = MRI.getType(DstReg);
2166 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2167 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
2168 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg);
2169
2170 MIRBuilder.buildTrunc(DstReg, ShrReg);
2172 return Legalized;
2173 }
2174 case TargetOpcode::G_BITREVERSE: {
2176
2177 Register DstReg = MI.getOperand(0).getReg();
2178 LLT Ty = MRI.getType(DstReg);
2179 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
2180
2181 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
2182 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2183 MI.getOperand(0).setReg(DstExt);
2185
2186 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits);
2187 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt);
2188 MIRBuilder.buildTrunc(DstReg, Shift);
2190 return Legalized;
2191 }
2192 case TargetOpcode::G_FREEZE:
2194 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2195 widenScalarDst(MI, WideTy);
2197 return Legalized;
2198
2199 case TargetOpcode::G_ABS:
2201 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
2202 widenScalarDst(MI, WideTy);
2204 return Legalized;
2205
2206 case TargetOpcode::G_ADD:
2207 case TargetOpcode::G_AND:
2208 case TargetOpcode::G_MUL:
2209 case TargetOpcode::G_OR:
2210 case TargetOpcode::G_XOR:
2211 case TargetOpcode::G_SUB:
2212 // Perform operation at larger width (any extension is fines here, high bits
2213 // don't affect the result) and then truncate the result back to the
2214 // original type.
2216 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2217 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2218 widenScalarDst(MI, WideTy);
2220 return Legalized;
2221
2222 case TargetOpcode::G_SBFX:
2223 case TargetOpcode::G_UBFX:
2225
2226 if (TypeIdx == 0) {
2227 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2228 widenScalarDst(MI, WideTy);
2229 } else {
2230 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2231 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT);
2232 }
2233
2235 return Legalized;
2236
2237 case TargetOpcode::G_SHL:
2239
2240 if (TypeIdx == 0) {
2241 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2242 widenScalarDst(MI, WideTy);
2243 } else {
2244 assert(TypeIdx == 1);
2245 // The "number of bits to shift" operand must preserve its value as an
2246 // unsigned integer:
2247 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2248 }
2249
2251 return Legalized;
2252
2253 case TargetOpcode::G_SDIV:
2254 case TargetOpcode::G_SREM:
2255 case TargetOpcode::G_SMIN:
2256 case TargetOpcode::G_SMAX:
2258 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
2259 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2260 widenScalarDst(MI, WideTy);
2262 return Legalized;
2263
2264 case TargetOpcode::G_SDIVREM:
2266 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2267 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
2268 widenScalarDst(MI, WideTy);
2269 widenScalarDst(MI, WideTy, 1);
2271 return Legalized;
2272
2273 case TargetOpcode::G_ASHR:
2274 case TargetOpcode::G_LSHR:
2276
2277 if (TypeIdx == 0) {
2278 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
2279 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2280
2281 widenScalarSrc(MI, WideTy, 1, CvtOp);
2282 widenScalarDst(MI, WideTy);
2283 } else {
2284 assert(TypeIdx == 1);
2285 // The "number of bits to shift" operand must preserve its value as an
2286 // unsigned integer:
2287 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2288 }
2289
2291 return Legalized;
2292 case TargetOpcode::G_UDIV:
2293 case TargetOpcode::G_UREM:
2294 case TargetOpcode::G_UMIN:
2295 case TargetOpcode::G_UMAX:
2297 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2298 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2299 widenScalarDst(MI, WideTy);
2301 return Legalized;
2302
2303 case TargetOpcode::G_UDIVREM:
2305 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2306 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT);
2307 widenScalarDst(MI, WideTy);
2308 widenScalarDst(MI, WideTy, 1);
2310 return Legalized;
2311
2312 case TargetOpcode::G_SELECT:
2314 if (TypeIdx == 0) {
2315 // Perform operation at larger width (any extension is fine here, high
2316 // bits don't affect the result) and then truncate the result back to the
2317 // original type.
2318 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2319 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
2320 widenScalarDst(MI, WideTy);
2321 } else {
2322 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
2323 // Explicit extension is required here since high bits affect the result.
2324 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
2325 }
2327 return Legalized;
2328
2329 case TargetOpcode::G_FPTOSI:
2330 case TargetOpcode::G_FPTOUI:
2332
2333 if (TypeIdx == 0)
2334 widenScalarDst(MI, WideTy);
2335 else
2336 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
2337
2339 return Legalized;
2340 case TargetOpcode::G_SITOFP:
2342
2343 if (TypeIdx == 0)
2344 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2345 else
2346 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
2347
2349 return Legalized;
2350 case TargetOpcode::G_UITOFP:
2352
2353 if (TypeIdx == 0)
2354 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2355 else
2356 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2357
2359 return Legalized;
2360 case TargetOpcode::G_LOAD:
2361 case TargetOpcode::G_SEXTLOAD:
2362 case TargetOpcode::G_ZEXTLOAD:
2364 widenScalarDst(MI, WideTy);
2366 return Legalized;
2367
2368 case TargetOpcode::G_STORE: {
2369 if (TypeIdx != 0)
2370 return UnableToLegalize;
2371
2372 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
2373 if (!Ty.isScalar())
2374 return UnableToLegalize;
2375
2377
2378 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
2379 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
2380 widenScalarSrc(MI, WideTy, 0, ExtType);
2381
2383 return Legalized;
2384 }
2385 case TargetOpcode::G_CONSTANT: {
2386 MachineOperand &SrcMO = MI.getOperand(1);
2388 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
2389 MRI.getType(MI.getOperand(0).getReg()));
2390 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
2391 ExtOpc == TargetOpcode::G_ANYEXT) &&
2392 "Illegal Extend");
2393 const APInt &SrcVal = SrcMO.getCImm()->getValue();
2394 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
2395 ? SrcVal.sext(WideTy.getSizeInBits())
2396 : SrcVal.zext(WideTy.getSizeInBits());
2398 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
2399
2400 widenScalarDst(MI, WideTy);
2402 return Legalized;
2403 }
2404 case TargetOpcode::G_FCONSTANT: {
2405 // To avoid changing the bits of the constant due to extension to a larger
2406 // type and then using G_FPTRUNC, we simply convert to a G_CONSTANT.
2407 MachineOperand &SrcMO = MI.getOperand(1);
2408 APInt Val = SrcMO.getFPImm()->getValueAPF().bitcastToAPInt();
2410 auto IntCst = MIRBuilder.buildConstant(MI.getOperand(0).getReg(), Val);
2411 widenScalarDst(*IntCst, WideTy, 0, TargetOpcode::G_TRUNC);
2412 MI.eraseFromParent();
2413 return Legalized;
2414 }
2415 case TargetOpcode::G_IMPLICIT_DEF: {
2417 widenScalarDst(MI, WideTy);
2419 return Legalized;
2420 }
2421 case TargetOpcode::G_BRCOND:
2423 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
2425 return Legalized;
2426
2427 case TargetOpcode::G_FCMP:
2429 if (TypeIdx == 0)
2430 widenScalarDst(MI, WideTy);
2431 else {
2432 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
2433 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
2434 }
2436 return Legalized;
2437
2438 case TargetOpcode::G_ICMP:
2440 if (TypeIdx == 0)
2441 widenScalarDst(MI, WideTy);
2442 else {
2443 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
2444 MI.getOperand(1).getPredicate()))
2445 ? TargetOpcode::G_SEXT
2446 : TargetOpcode::G_ZEXT;
2447 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
2448 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
2449 }
2451 return Legalized;
2452
2453 case TargetOpcode::G_PTR_ADD:
2454 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
2456 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2458 return Legalized;
2459
2460 case TargetOpcode::G_PHI: {
2461 assert(TypeIdx == 0 && "Expecting only Idx 0");
2462
2464 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
2465 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2467 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
2468 }
2469
2470 MachineBasicBlock &MBB = *MI.getParent();
2472 widenScalarDst(MI, WideTy);
2474 return Legalized;
2475 }
2476 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2477 if (TypeIdx == 0) {
2478 Register VecReg = MI.getOperand(1).getReg();
2479 LLT VecTy = MRI.getType(VecReg);
2481
2483 MI, LLT::vector(VecTy.getElementCount(), WideTy.getSizeInBits()), 1,
2484 TargetOpcode::G_ANYEXT);
2485
2486 widenScalarDst(MI, WideTy, 0);
2488 return Legalized;
2489 }
2490
2491 if (TypeIdx != 2)
2492 return UnableToLegalize;
2494 // TODO: Probably should be zext
2495 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2497 return Legalized;
2498 }
2499 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2500 if (TypeIdx == 1) {
2502
2503 Register VecReg = MI.getOperand(1).getReg();
2504 LLT VecTy = MRI.getType(VecReg);
2505 LLT WideVecTy = LLT::vector(VecTy.getElementCount(), WideTy);
2506
2507 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT);
2508 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2509 widenScalarDst(MI, WideVecTy, 0);
2511 return Legalized;
2512 }
2513
2514 if (TypeIdx == 2) {
2516 // TODO: Probably should be zext
2517 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
2519 return Legalized;
2520 }
2521
2522 return UnableToLegalize;
2523 }
2524 case TargetOpcode::G_FADD:
2525 case TargetOpcode::G_FMUL:
2526 case TargetOpcode::G_FSUB:
2527 case TargetOpcode::G_FMA:
2528 case TargetOpcode::G_FMAD:
2529 case TargetOpcode::G_FNEG:
2530 case TargetOpcode::G_FABS:
2531 case TargetOpcode::G_FCANONICALIZE:
2532 case TargetOpcode::G_FMINNUM:
2533 case TargetOpcode::G_FMAXNUM:
2534 case TargetOpcode::G_FMINNUM_IEEE:
2535 case TargetOpcode::G_FMAXNUM_IEEE:
2536 case TargetOpcode::G_FMINIMUM:
2537 case TargetOpcode::G_FMAXIMUM:
2538 case TargetOpcode::G_FDIV:
2539 case TargetOpcode::G_FREM:
2540 case TargetOpcode::G_FCEIL:
2541 case TargetOpcode::G_FFLOOR:
2542 case TargetOpcode::G_FCOS:
2543 case TargetOpcode::G_FSIN:
2544 case TargetOpcode::G_FLOG10:
2545 case TargetOpcode::G_FLOG:
2546 case TargetOpcode::G_FLOG2:
2547 case TargetOpcode::G_FRINT:
2548 case TargetOpcode::G_FNEARBYINT:
2549 case TargetOpcode::G_FSQRT:
2550 case TargetOpcode::G_FEXP:
2551 case TargetOpcode::G_FEXP2:
2552 case TargetOpcode::G_FPOW:
2553 case TargetOpcode::G_INTRINSIC_TRUNC:
2554 case TargetOpcode::G_INTRINSIC_ROUND:
2555 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2556 assert(TypeIdx == 0);
2558
2559 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
2560 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
2561
2562 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2564 return Legalized;
2565 case TargetOpcode::G_FPOWI: {
2566 if (TypeIdx != 0)
2567 return UnableToLegalize;
2569 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
2570 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2572 return Legalized;
2573 }
2574 case TargetOpcode::G_INTTOPTR:
2575 if (TypeIdx != 1)
2576 return UnableToLegalize;
2577
2579 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2581 return Legalized;
2582 case TargetOpcode::G_PTRTOINT:
2583 if (TypeIdx != 0)
2584 return UnableToLegalize;
2585
2587 widenScalarDst(MI, WideTy, 0);
2589 return Legalized;
2590 case TargetOpcode::G_BUILD_VECTOR: {
2592
2593 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
2594 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
2595 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
2596
2597 // Avoid changing the result vector type if the source element type was
2598 // requested.
2599 if (TypeIdx == 1) {
2600 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
2601 } else {
2602 widenScalarDst(MI, WideTy, 0);
2603 }
2604
2606 return Legalized;
2607 }
2608 case TargetOpcode::G_SEXT_INREG:
2609 if (TypeIdx != 0)
2610 return UnableToLegalize;
2611
2613 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2614 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
2616 return Legalized;
2617 case TargetOpcode::G_PTRMASK: {
2618 if (TypeIdx != 1)
2619 return UnableToLegalize;
2621 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2623 return Legalized;
2624 }
2625 }
2626}
2627
2629 MachineIRBuilder &B, Register Src, LLT Ty) {
2630 auto Unmerge = B.buildUnmerge(Ty, Src);
2631 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
2632 Pieces.push_back(Unmerge.getReg(I));
2633}
2634
2637 Register Dst = MI.getOperand(0).getReg();
2638
2641
2642 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
2643 LLT AddrPtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
2644 Align Alignment = Align(DL.getABITypeAlign(
2646
2648 AddrPtrTy, MF.getConstantPool()->getConstantPoolIndex(
2649 MI.getOperand(1).getFPImm(), Alignment));
2650
2653 MRI.getType(Dst), Alignment);
2654
2655 MIRBuilder.buildLoadInstr(TargetOpcode::G_LOAD, Dst, Addr, *MMO);
2656 MI.eraseFromParent();
2657
2658 return Legalized;
2659}
2660
2663 Register Dst = MI.getOperand(0).getReg();
2664 Register Src = MI.getOperand(1).getReg();
2665 LLT DstTy = MRI.getType(Dst);
2666 LLT SrcTy = MRI.getType(Src);
2667
2668 if (SrcTy.isVector()) {
2669 LLT SrcEltTy = SrcTy.getElementType();
2671
2672 if (DstTy.isVector()) {
2673 int NumDstElt = DstTy.getNumElements();
2674 int NumSrcElt = SrcTy.getNumElements();
2675
2676 LLT DstEltTy = DstTy.getElementType();
2677 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
2678 LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
2679
2680 // If there's an element size mismatch, insert intermediate casts to match
2681 // the result element type.
2682 if (NumSrcElt < NumDstElt) { // Source element type is larger.
2683 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
2684 //
2685 // =>
2686 //
2687 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
2688 // %3:_(<2 x s8>) = G_BITCAST %2
2689 // %4:_(<2 x s8>) = G_BITCAST %3
2690 // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4
2691 DstCastTy = LLT::fixed_vector(NumDstElt / NumSrcElt, DstEltTy);
2692 SrcPartTy = SrcEltTy;
2693 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
2694 //
2695 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
2696 //
2697 // =>
2698 //
2699 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
2700 // %3:_(s16) = G_BITCAST %2
2701 // %4:_(s16) = G_BITCAST %3
2702 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
2703 SrcPartTy = LLT::fixed_vector(NumSrcElt / NumDstElt, SrcEltTy);
2704 DstCastTy = DstEltTy;
2705 }
2706
2707 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy);
2708 for (Register &SrcReg : SrcRegs)
2709 SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0);
2710 } else
2711 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy);
2712
2713 MIRBuilder.buildMergeLikeInstr(Dst, SrcRegs);
2714 MI.eraseFromParent();
2715 return Legalized;
2716 }
2717
2718 if (DstTy.isVector()) {
2720 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType());
2721 MIRBuilder.buildMergeLikeInstr(Dst, SrcRegs);
2722 MI.eraseFromParent();
2723 return Legalized;
2724 }
2725
2726 return UnableToLegalize;
2727}
2728
2729/// Figure out the bit offset into a register when coercing a vector index for
2730/// the wide element type. This is only for the case when promoting vector to
2731/// one with larger elements.
2732//
2733///
2734/// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
2735/// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
2737 Register Idx,
2738 unsigned NewEltSize,
2739 unsigned OldEltSize) {
2740 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2741 LLT IdxTy = B.getMRI()->getType(Idx);
2742
2743 // Now figure out the amount we need to shift to get the target bits.
2744 auto OffsetMask = B.buildConstant(
2745 IdxTy, ~(APInt::getAllOnes(IdxTy.getSizeInBits()) << Log2EltRatio));
2746 auto OffsetIdx = B.buildAnd(IdxTy, Idx, OffsetMask);
2747 return B.buildShl(IdxTy, OffsetIdx,
2748 B.buildConstant(IdxTy, Log2_32(OldEltSize))).getReg(0);
2749}
2750
2751/// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
2752/// is casting to a vector with a smaller element size, perform multiple element
2753/// extracts and merge the results. If this is coercing to a vector with larger
2754/// elements, index the bitcasted vector and extract the target element with bit
2755/// operations. This is intended to force the indexing in the native register
2756/// size for architectures that can dynamically index the register file.
2759 LLT CastTy) {
2760 if (TypeIdx != 1)
2761 return UnableToLegalize;
2762
2763 Register Dst = MI.getOperand(0).getReg();
2764 Register SrcVec = MI.getOperand(1).getReg();
2765 Register Idx = MI.getOperand(2).getReg();
2766 LLT SrcVecTy = MRI.getType(SrcVec);
2767 LLT IdxTy = MRI.getType(Idx);
2768
2769 LLT SrcEltTy = SrcVecTy.getElementType();
2770 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
2771 unsigned OldNumElts = SrcVecTy.getNumElements();
2772
2773 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
2774 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
2775
2776 const unsigned NewEltSize = NewEltTy.getSizeInBits();
2777 const unsigned OldEltSize = SrcEltTy.getSizeInBits();
2778 if (NewNumElts > OldNumElts) {
2779 // Decreasing the vector element size
2780 //
2781 // e.g. i64 = extract_vector_elt x:v2i64, y:i32
2782 // =>
2783 // v4i32:castx = bitcast x:v2i64
2784 //
2785 // i64 = bitcast
2786 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
2787 // (i32 (extract_vector_elt castx, (2 * y + 1)))
2788 //
2789 if (NewNumElts % OldNumElts != 0)
2790 return UnableToLegalize;
2791
2792 // Type of the intermediate result vector.
2793 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
2794 LLT MidTy =
2795 LLT::scalarOrVector(ElementCount::getFixed(NewEltsPerOldElt), NewEltTy);
2796
2797 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt);
2798
2799 SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
2800 auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK);
2801
2802 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
2803 auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I);
2804 auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset);
2805 auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx);
2806 NewOps[I] = Elt.getReg(0);
2807 }
2808
2809 auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps);
2810 MIRBuilder.buildBitcast(Dst, NewVec);
2811 MI.eraseFromParent();
2812 return Legalized;
2813 }
2814
2815 if (NewNumElts < OldNumElts) {
2816 if (NewEltSize % OldEltSize != 0)
2817 return UnableToLegalize;
2818
2819 // This only depends on powers of 2 because we use bit tricks to figure out
2820 // the bit offset we need to shift to get the target element. A general
2821 // expansion could emit division/multiply.
2822 if (!isPowerOf2_32(NewEltSize / OldEltSize))
2823 return UnableToLegalize;
2824
2825 // Increasing the vector element size.
2826 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
2827 //
2828 // =>
2829 //
2830 // %cast = G_BITCAST %vec
2831 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
2832 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
2833 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
2834 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
2835 // %elt_bits = G_LSHR %wide_elt, %offset_bits
2836 // %elt = G_TRUNC %elt_bits
2837
2838 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2839 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
2840
2841 // Divide to get the index in the wider element type.
2842 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
2843
2844 Register WideElt = CastVec;
2845 if (CastTy.isVector()) {
2846 WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
2847 ScaledIdx).getReg(0);
2848 }
2849
2850 // Compute the bit offset into the register of the target element.
2852 MIRBuilder, Idx, NewEltSize, OldEltSize);
2853
2854 // Shift the wide element to get the target element.
2855 auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits);
2856 MIRBuilder.buildTrunc(Dst, ExtractedBits);
2857 MI.eraseFromParent();
2858 return Legalized;
2859 }
2860
2861 return UnableToLegalize;
2862}
2863
2864/// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
2865/// TargetReg, while preserving other bits in \p TargetReg.
2866///
2867/// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
2869 Register TargetReg, Register InsertReg,
2870 Register OffsetBits) {
2871 LLT TargetTy = B.getMRI()->getType(TargetReg);
2872 LLT InsertTy = B.getMRI()->getType(InsertReg);
2873 auto ZextVal = B.buildZExt(TargetTy, InsertReg);
2874 auto ShiftedInsertVal = B.buildShl(TargetTy, ZextVal, OffsetBits);
2875
2876 // Produce a bitmask of the value to insert
2877 auto EltMask = B.buildConstant(
2878 TargetTy, APInt::getLowBitsSet(TargetTy.getSizeInBits(),
2879 InsertTy.getSizeInBits()));
2880 // Shift it into position
2881 auto ShiftedMask = B.buildShl(TargetTy, EltMask, OffsetBits);
2882 auto InvShiftedMask = B.buildNot(TargetTy, ShiftedMask);
2883
2884 // Clear out the bits in the wide element
2885 auto MaskedOldElt = B.buildAnd(TargetTy, TargetReg, InvShiftedMask);
2886
2887 // The value to insert has all zeros already, so stick it into the masked
2888 // wide element.
2889 return B.buildOr(TargetTy, MaskedOldElt, ShiftedInsertVal).getReg(0);
2890}
2891
2892/// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
2893/// is increasing the element size, perform the indexing in the target element
2894/// type, and use bit operations to insert at the element position. This is
2895/// intended for architectures that can dynamically index the register file and
2896/// want to force indexing in the native register size.
2899 LLT CastTy) {
2900 if (TypeIdx != 0)
2901 return UnableToLegalize;
2902
2903 Register Dst = MI.getOperand(0).getReg();
2904 Register SrcVec = MI.getOperand(1).getReg();
2905 Register Val = MI.getOperand(2).getReg();
2906 Register Idx = MI.getOperand(3).getReg();
2907
2908 LLT VecTy = MRI.getType(Dst);
2909 LLT IdxTy = MRI.getType(Idx);
2910
2911 LLT VecEltTy = VecTy.getElementType();
2912 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
2913 const unsigned NewEltSize = NewEltTy.getSizeInBits();
2914 const unsigned OldEltSize = VecEltTy.getSizeInBits();
2915
2916 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
2917 unsigned OldNumElts = VecTy.getNumElements();
2918
2919 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
2920 if (NewNumElts < OldNumElts) {
2921 if (NewEltSize % OldEltSize != 0)
2922 return UnableToLegalize;
2923
2924 // This only depends on powers of 2 because we use bit tricks to figure out
2925 // the bit offset we need to shift to get the target element. A general
2926 // expansion could emit division/multiply.
2927 if (!isPowerOf2_32(NewEltSize / OldEltSize))
2928 return UnableToLegalize;
2929
2930 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2931 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
2932
2933 // Divide to get the index in the wider element type.
2934 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
2935
2936 Register ExtractedElt = CastVec;
2937 if (CastTy.isVector()) {
2938 ExtractedElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
2939 ScaledIdx).getReg(0);
2940 }
2941
2942 // Compute the bit offset into the register of the target element.
2944 MIRBuilder, Idx, NewEltSize, OldEltSize);
2945
2946 Register InsertedElt = buildBitFieldInsert(MIRBuilder, ExtractedElt,
2947 Val, OffsetBits);
2948 if (CastTy.isVector()) {
2950 CastTy, CastVec, InsertedElt, ScaledIdx).getReg(0);
2951 }
2952
2953 MIRBuilder.buildBitcast(Dst, InsertedElt);
2954 MI.eraseFromParent();
2955 return Legalized;
2956 }
2957
2958 return UnableToLegalize;
2959}
2960
2962 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
2963 Register DstReg = LoadMI.getDstReg();
2964 Register PtrReg = LoadMI.getPointerReg();
2965 LLT DstTy = MRI.getType(DstReg);
2966 MachineMemOperand &MMO = LoadMI.getMMO();
2967 LLT MemTy = MMO.getMemoryType();
2969
2970 unsigned MemSizeInBits = MemTy.getSizeInBits();
2971 unsigned MemStoreSizeInBits = 8 * MemTy.getSizeInBytes();
2972
2973 if (MemSizeInBits != MemStoreSizeInBits) {
2974 if (MemTy.isVector())
2975 return UnableToLegalize;
2976
2977 // Promote to a byte-sized load if not loading an integral number of
2978 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
2979 LLT WideMemTy = LLT::scalar(MemStoreSizeInBits);
2980 MachineMemOperand *NewMMO =
2981 MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), WideMemTy);
2982
2983 Register LoadReg = DstReg;
2984 LLT LoadTy = DstTy;
2985
2986 // If this wasn't already an extending load, we need to widen the result
2987 // register to avoid creating a load with a narrower result than the source.
2988 if (MemStoreSizeInBits > DstTy.getSizeInBits()) {
2989 LoadTy = WideMemTy;
2990 LoadReg = MRI.createGenericVirtualRegister(WideMemTy);
2991 }
2992
2993 if (isa<GSExtLoad>(LoadMI)) {
2994 auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO);
2995 MIRBuilder.buildSExtInReg(LoadReg, NewLoad, MemSizeInBits);
2996 } else if (isa<GZExtLoad>(LoadMI) || WideMemTy == LoadTy) {
2997 auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO);
2998 // The extra bits are guaranteed to be zero, since we stored them that
2999 // way. A zext load from Wide thus automatically gives zext from MemVT.
3000 MIRBuilder.buildAssertZExt(LoadReg, NewLoad, MemSizeInBits);
3001 } else {
3002 MIRBuilder.buildLoad(LoadReg, PtrReg, *NewMMO);
3003 }
3004
3005 if (DstTy != LoadTy)
3006 MIRBuilder.buildTrunc(DstReg, LoadReg);
3007
3008 LoadMI.eraseFromParent();
3009 return Legalized;
3010 }
3011
3012 // Big endian lowering not implemented.
3014 return UnableToLegalize;
3015
3016 // This load needs splitting into power of 2 sized loads.
3017 //
3018 // Our strategy here is to generate anyextending loads for the smaller
3019 // types up to next power-2 result type, and then combine the two larger
3020 // result values together, before truncating back down to the non-pow-2
3021 // type.
3022 // E.g. v1 = i24 load =>
3023 // v2 = i32 zextload (2 byte)
3024 // v3 = i32 load (1 byte)
3025 // v4 = i32 shl v3, 16
3026 // v5 = i32 or v4, v2
3027 // v1 = i24 trunc v5
3028 // By doing this we generate the correct truncate which should get
3029 // combined away as an artifact with a matching extend.
3030
3031 uint64_t LargeSplitSize, SmallSplitSize;
3032
3033 if (!isPowerOf2_32(MemSizeInBits)) {
3034 // This load needs splitting into power of 2 sized loads.
3035 LargeSplitSize = llvm::bit_floor(MemSizeInBits);
3036 SmallSplitSize = MemSizeInBits - LargeSplitSize;
3037 } else {
3038 // This is already a power of 2, but we still need to split this in half.
3039 //
3040 // Assume we're being asked to decompose an unaligned load.
3041 // TODO: If this requires multiple splits, handle them all at once.
3042 auto &Ctx = MF.getFunction().getContext();
3043 if (TLI.allowsMemoryAccess(Ctx, MIRBuilder.getDataLayout(), MemTy, MMO))
3044 return UnableToLegalize;
3045
3046 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3047 }
3048
3049 if (MemTy.isVector()) {
3050 // TODO: Handle vector extloads
3051 if (MemTy != DstTy)
3052 return UnableToLegalize;
3053
3054 // TODO: We can do better than scalarizing the vector and at least split it
3055 // in half.
3056 return reduceLoadStoreWidth(LoadMI, 0, DstTy.getElementType());
3057 }
3058
3059 MachineMemOperand *LargeMMO =
3060 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
3061 MachineMemOperand *SmallMMO =
3062 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
3063
3064 LLT PtrTy = MRI.getType(PtrReg);
3065 unsigned AnyExtSize = PowerOf2Ceil(DstTy.getSizeInBits());
3066 LLT AnyExtTy = LLT::scalar(AnyExtSize);
3067 auto LargeLoad = MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, AnyExtTy,
3068 PtrReg, *LargeMMO);
3069
3070 auto OffsetCst = MIRBuilder.buildConstant(LLT::scalar(PtrTy.getSizeInBits()),
3071 LargeSplitSize / 8);
3072 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
3073 auto SmallPtr = MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst);
3074 auto SmallLoad = MIRBuilder.buildLoadInstr(LoadMI.getOpcode(), AnyExtTy,
3075 SmallPtr, *SmallMMO);
3076
3077 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize);
3078 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt);
3079
3080 if (AnyExtTy == DstTy)
3081 MIRBuilder.buildOr(DstReg, Shift, LargeLoad);
3082 else if (AnyExtTy.getSizeInBits() != DstTy.getSizeInBits()) {
3083 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
3084 MIRBuilder.buildTrunc(DstReg, {Or});
3085 } else {
3086 assert(DstTy.isPointer() && "expected pointer");
3087 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
3088
3089 // FIXME: We currently consider this to be illegal for non-integral address
3090 // spaces, but we need still need a way to reinterpret the bits.
3091 MIRBuilder.buildIntToPtr(DstReg, Or);
3092 }
3093
3094 LoadMI.eraseFromParent();
3095 return Legalized;
3096}
3097
3099 // Lower a non-power of 2 store into multiple pow-2 stores.
3100 // E.g. split an i24 store into an i16 store + i8 store.
3101 // We do this by first extending the stored value to the next largest power
3102 // of 2 type, and then using truncating stores to store the components.
3103 // By doing this, likewise with G_LOAD, generate an extend that can be
3104 // artifact-combined away instead of leaving behind extracts.
3105 Register SrcReg = StoreMI.getValueReg();
3106 Register PtrReg = StoreMI.getPointerReg();
3107 LLT SrcTy = MRI.getType(SrcReg);
3109 MachineMemOperand &MMO = **StoreMI.memoperands_begin();
3110 LLT MemTy = MMO.getMemoryType();
3111
3112 unsigned StoreWidth = MemTy.getSizeInBits();
3113 unsigned StoreSizeInBits = 8 * MemTy.getSizeInBytes();
3114
3115 if (StoreWidth != StoreSizeInBits) {
3116 if (SrcTy.isVector())
3117 return UnableToLegalize;
3118
3119 // Promote to a byte-sized store with upper bits zero if not
3120 // storing an integral number of bytes. For example, promote
3121 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
3122 LLT WideTy = LLT::scalar(StoreSizeInBits);
3123
3124 if (StoreSizeInBits > SrcTy.getSizeInBits()) {
3125 // Avoid creating a store with a narrower source than result.
3126 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0);
3127 SrcTy = WideTy;
3128 }
3129
3130 auto ZextInReg = MIRBuilder.buildZExtInReg(SrcTy, SrcReg, StoreWidth);
3131
3132 MachineMemOperand *NewMMO =
3133 MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), WideTy);
3134 MIRBuilder.buildStore(ZextInReg, PtrReg, *NewMMO);
3135 StoreMI.eraseFromParent();
3136 return Legalized;
3137 }
3138
3139 if (MemTy.isVector()) {
3140 // TODO: Handle vector trunc stores
3141 if (MemTy != SrcTy)
3142 return UnableToLegalize;
3143
3144 // TODO: We can do better than scalarizing the vector and at least split it
3145 // in half.
3146 return reduceLoadStoreWidth(StoreMI, 0, SrcTy.getElementType());
3147 }
3148
3149 unsigned MemSizeInBits = MemTy.getSizeInBits();
3150 uint64_t LargeSplitSize, SmallSplitSize;
3151
3152 if (!isPowerOf2_32(MemSizeInBits)) {
3153 LargeSplitSize = llvm::bit_floor<uint64_t>(MemTy.getSizeInBits());
3154 SmallSplitSize = MemTy.getSizeInBits() - LargeSplitSize;
3155 } else {
3156 auto &Ctx = MF.getFunction().getContext();
3157 if (TLI.allowsMemoryAccess(Ctx, MIRBuilder.getDataLayout(), MemTy, MMO))
3158 return UnableToLegalize; // Don't know what we're being asked to do.
3159
3160 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3161 }
3162
3163 // Extend to the next pow-2. If this store was itself the result of lowering,
3164 // e.g. an s56 store being broken into s32 + s24, we might have a stored type
3165 // that's wider than the stored size.
3166 unsigned AnyExtSize = PowerOf2Ceil(MemTy.getSizeInBits());
3167 const LLT NewSrcTy = LLT::scalar(AnyExtSize);
3168
3169 if (SrcTy.isPointer()) {
3170 const LLT IntPtrTy = LLT::scalar(SrcTy.getSizeInBits());
3171 SrcReg = MIRBuilder.buildPtrToInt(IntPtrTy, SrcReg).getReg(0);
3172 }
3173
3174 auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(NewSrcTy, SrcReg);
3175
3176 // Obtain the smaller value by shifting away the larger value.
3177 auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, LargeSplitSize);
3178 auto SmallVal = MIRBuilder.buildLShr(NewSrcTy, ExtVal, ShiftAmt);
3179
3180 // Generate the PtrAdd and truncating stores.
3181 LLT PtrTy = MRI.getType(PtrReg);
3182 auto OffsetCst = MIRBuilder.buildConstant(
3183 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8);
3184 auto SmallPtr =
3185 MIRBuilder.buildPtrAdd(PtrTy, PtrReg, OffsetCst);
3186
3187 MachineMemOperand *LargeMMO =
3188 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
3189 MachineMemOperand *SmallMMO =
3190 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
3191 MIRBuilder.buildStore(ExtVal, PtrReg, *LargeMMO);
3192 MIRBuilder.buildStore(SmallVal, SmallPtr, *SmallMMO);
3193 StoreMI.eraseFromParent();
3194 return Legalized;
3195}
3196
3198LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
3199 switch (MI.getOpcode()) {
3200 case TargetOpcode::G_LOAD: {
3201 if (TypeIdx != 0)
3202 return UnableToLegalize;
3203 MachineMemOperand &MMO = **MI.memoperands_begin();
3204
3205 // Not sure how to interpret a bitcast of an extending load.
3206 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
3207 return UnableToLegalize;
3208
3210 bitcastDst(MI, CastTy, 0);
3211 MMO.setType(CastTy);
3213 return Legalized;
3214 }
3215 case TargetOpcode::G_STORE: {
3216 if (TypeIdx != 0)
3217 return UnableToLegalize;
3218
3219 MachineMemOperand &MMO = **MI.memoperands_begin();
3220
3221 // Not sure how to interpret a bitcast of a truncating store.
3222 if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits())
3223 return UnableToLegalize;
3224
3226 bitcastSrc(MI, CastTy, 0);
3227 MMO.setType(CastTy);
3229 return Legalized;
3230 }
3231 case TargetOpcode::G_SELECT: {
3232 if (TypeIdx != 0)
3233 return UnableToLegalize;
3234
3235 if (MRI.getType(MI.getOperand(1).getReg()).isVector()) {
3236 LLVM_DEBUG(
3237 dbgs() << "bitcast action not implemented for vector select\n");
3238 return UnableToLegalize;
3239 }
3240
3242 bitcastSrc(MI, CastTy, 2);
3243 bitcastSrc(MI, CastTy, 3);
3244 bitcastDst(MI, CastTy, 0);
3246 return Legalized;
3247 }
3248 case TargetOpcode::G_AND:
3249 case TargetOpcode::G_OR:
3250 case TargetOpcode::G_XOR: {
3252 bitcastSrc(MI, CastTy, 1);
3253 bitcastSrc(MI, CastTy, 2);
3254 bitcastDst(MI, CastTy, 0);
3256 return Legalized;
3257 }
3258 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3259 return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
3260 case TargetOpcode::G_INSERT_VECTOR_ELT:
3261 return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
3262 default:
3263 return UnableToLegalize;
3264 }
3265}
3266
3267// Legalize an instruction by changing the opcode in place.
3268void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
3270 MI.setDesc(MIRBuilder.getTII().get(NewOpcode));
3272}
3273
3275LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
3276 using namespace TargetOpcode;
3277
3278 switch(MI.getOpcode()) {
3279 default:
3280 return UnableToLegalize;
3281 case TargetOpcode::G_FCONSTANT:
3282 return lowerFConstant(MI);
3283 case TargetOpcode::G_BITCAST:
3284 return lowerBitcast(MI);
3285 case TargetOpcode::G_SREM:
3286 case TargetOpcode::G_UREM: {
3287 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3288 auto Quot =
3289 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty},
3290 {MI.getOperand(1), MI.getOperand(2)});
3291
3292 auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2));
3293 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod);
3294 MI.eraseFromParent();
3295 return Legalized;
3296 }
3297 case TargetOpcode::G_SADDO:
3298 case TargetOpcode::G_SSUBO:
3299 return lowerSADDO_SSUBO(MI);
3300 case TargetOpcode::G_UMULH:
3301 case TargetOpcode::G_SMULH:
3302 return lowerSMULH_UMULH(MI);
3303 case TargetOpcode::G_SMULO:
3304 case TargetOpcode::G_UMULO: {
3305 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
3306 // result.
3307 Register Res = MI.getOperand(0).getReg();
3308 Register Overflow = MI.getOperand(1).getReg();
3309 Register LHS = MI.getOperand(2).getReg();
3310 Register RHS = MI.getOperand(3).getReg();
3311 LLT Ty = MRI.getType(Res);
3312
3313 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
3314 ? TargetOpcode::G_SMULH
3315 : TargetOpcode::G_UMULH;
3316
3318 const auto &TII = MIRBuilder.getTII();
3319 MI.setDesc(TII.get(TargetOpcode::G_MUL));
3320 MI.removeOperand(1);
3322
3323 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS});
3324 auto Zero = MIRBuilder.buildConstant(Ty, 0);
3325
3326 // Move insert point forward so we can use the Res register if needed.
3328
3329 // For *signed* multiply, overflow is detected by checking:
3330 // (hi != (lo >> bitwidth-1))
3331 if (Opcode == TargetOpcode::G_SMULH) {
3332 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1);
3333 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt);
3334 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
3335 } else {
3336 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
3337 }
3338 return Legalized;
3339 }
3340 case TargetOpcode::G_FNEG: {
3341 Register Res = MI.getOperand(0).getReg();
3342 LLT Ty = MRI.getType(Res);
3343
3344 // TODO: Handle vector types once we are able to
3345 // represent them.
3346 if (Ty.isVector())
3347 return UnableToLegalize;
3348 auto SignMask =
3350 Register SubByReg = MI.getOperand(1).getReg();
3351 MIRBuilder.buildXor(Res, SubByReg, SignMask);
3352 MI.eraseFromParent();
3353 return Legalized;
3354 }
3355 case TargetOpcode::G_FSUB:
3356 case TargetOpcode::G_STRICT_FSUB: {
3357 Register Res = MI.getOperand(0).getReg();
3358 LLT Ty = MRI.getType(Res);
3359
3360 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
3361 // First, check if G_FNEG is marked as Lower. If so, we may
3362 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
3363 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
3364 return UnableToLegalize;
3365 Register LHS = MI.getOperand(1).getReg();
3366 Register RHS = MI.getOperand(2).getReg();
3367 auto Neg = MIRBuilder.buildFNeg(Ty, RHS);
3368
3369 if (MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
3370 MIRBuilder.buildStrictFAdd(Res, LHS, Neg, MI.getFlags());
3371 else
3372 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags());
3373
3374 MI.eraseFromParent();
3375 return Legalized;
3376 }
3377 case TargetOpcode::G_FMAD:
3378 return lowerFMad(MI);
3379 case TargetOpcode::G_FFLOOR:
3380 return lowerFFloor(MI);
3381 case TargetOpcode::G_INTRINSIC_ROUND:
3382 return lowerIntrinsicRound(MI);
3383 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
3384 // Since round even is the assumed rounding mode for unconstrained FP
3385 // operations, rint and roundeven are the same operation.
3386 changeOpcode(MI, TargetOpcode::G_FRINT);
3387 return Legalized;
3388 }
3389 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
3390 Register OldValRes = MI.getOperand(0).getReg();
3391 Register SuccessRes = MI.getOperand(1).getReg();
3392 Register Addr = MI.getOperand(2).getReg();
3393 Register CmpVal = MI.getOperand(3).getReg();
3394 Register NewVal = MI.getOperand(4).getReg();
3395 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
3396 **MI.memoperands_begin());
3397 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
3398 MI.eraseFromParent();
3399 return Legalized;
3400 }
3401 case TargetOpcode::G_LOAD:
3402 case TargetOpcode::G_SEXTLOAD:
3403 case TargetOpcode::G_ZEXTLOAD:
3404 return lowerLoad(cast<GAnyLoad>(MI));
3405 case TargetOpcode::G_STORE:
3406 return lowerStore(cast<GStore>(MI));
3407 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
3408 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
3409 case TargetOpcode::G_CTLZ:
3410 case TargetOpcode::G_CTTZ:
3411 case TargetOpcode::G_CTPOP:
3412 return lowerBitCount(MI);
3413 case G_UADDO: {
3414 Register Res = MI.getOperand(0).getReg();
3415 Register CarryOut = MI.getOperand(1).getReg();
3416 Register LHS = MI.getOperand(2).getReg();
3417 Register RHS = MI.getOperand(3).getReg();
3418
3419 MIRBuilder.buildAdd(Res, LHS, RHS);
3420 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
3421
3422 MI.eraseFromParent();
3423 return Legalized;
3424 }
3425 case G_UADDE: {
3426 Register Res = MI.getOperand(0).getReg();
3427 Register CarryOut = MI.getOperand(1).getReg();
3428 Register LHS = MI.getOperand(2).getReg();
3429 Register RHS = MI.getOperand(3).getReg();
3430 Register CarryIn = MI.getOperand(4).getReg();
3431 LLT Ty = MRI.getType(Res);
3432
3433 auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS);
3434 auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn);
3435 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
3436 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
3437
3438 MI.eraseFromParent();
3439 return Legalized;
3440 }
3441 case G_USUBO: {
3442 Register Res = MI.getOperand(0).getReg();
3443 Register BorrowOut = MI.getOperand(1).getReg();
3444 Register LHS = MI.getOperand(2).getReg();
3445 Register RHS = MI.getOperand(3).getReg();
3446
3447 MIRBuilder.buildSub(Res, LHS, RHS);
3449
3450 MI.eraseFromParent();
3451 return Legalized;
3452 }
3453 case G_USUBE: {
3454 Register Res = MI.getOperand(0).getReg();
3455 Register BorrowOut = MI.getOperand(1).getReg();
3456 Register LHS = MI.getOperand(2).getReg();
3457 Register RHS = MI.getOperand(3).getReg();
3458 Register BorrowIn = MI.getOperand(4).getReg();
3459 const LLT CondTy = MRI.getType(BorrowOut);
3460 const LLT Ty = MRI.getType(Res);
3461
3462 auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS);
3463 auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn);
3464 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
3465
3466 auto LHS_EQ_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, LHS, RHS);
3467 auto LHS_ULT_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, LHS, RHS);
3468 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
3469
3470 MI.eraseFromParent();
3471 return Legalized;
3472 }
3473 case G_UITOFP:
3474 return lowerUITOFP(MI);
3475 case G_SITOFP:
3476 return lowerSITOFP(MI);
3477 case G_FPTOUI:
3478 return lowerFPTOUI(MI);
3479 case G_FPTOSI:
3480 return lowerFPTOSI(MI);
3481 case G_FPTRUNC:
3482 return lowerFPTRUNC(MI);
3483 case G_FPOWI:
3484 return lowerFPOWI(MI);
3485 case G_SMIN:
3486 case G_SMAX:
3487 case G_UMIN:
3488 case G_UMAX:
3489 return lowerMinMax(MI);
3490 case G_FCOPYSIGN:
3491 return lowerFCopySign(MI);
3492 case G_FMINNUM:
3493 case G_FMAXNUM:
3494 return lowerFMinNumMaxNum(MI);
3495 case G_MERGE_VALUES:
3496 return lowerMergeValues(MI);
3497 case G_UNMERGE_VALUES:
3498 return lowerUnmergeValues(MI);
3499 case TargetOpcode::G_SEXT_INREG: {
3500 assert(MI.getOperand(2).isImm() && "Expected immediate");
3501 int64_t SizeInBits = MI.getOperand(2).getImm();
3502
3503 Register DstReg = MI.getOperand(0).getReg();
3504 Register SrcReg = MI.getOperand(1).getReg();
3505 LLT DstTy = MRI.getType(DstReg);
3506 Register TmpRes = MRI.createGenericVirtualRegister(DstTy);
3507
3508 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits);
3509 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0));
3510 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0));
3511 MI.eraseFromParent();
3512 return Legalized;
3513 }
3514 case G_EXTRACT_VECTOR_ELT:
3515 case G_INSERT_VECTOR_ELT:
3517 case G_SHUFFLE_VECTOR:
3518 return lowerShuffleVector(MI);
3519 case G_DYN_STACKALLOC:
3520 return lowerDynStackAlloc(MI);
3521 case G_EXTRACT:
3522 return lowerExtract(MI);
3523 case G_INSERT:
3524 return lowerInsert(MI);
3525 case G_BSWAP:
3526 return lowerBswap(MI);
3527 case G_BITREVERSE:
3528 return lowerBitreverse(MI);
3529 case G_READ_REGISTER:
3530 case G_WRITE_REGISTER:
3531 return lowerReadWriteRegister(MI);
3532 case G_UADDSAT:
3533 case G_USUBSAT: {
3534 // Try to make a reasonable guess about which lowering strategy to use. The
3535 // target can override this with custom lowering and calling the
3536 // implementation functions.
3537 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3538 if (LI.isLegalOrCustom({G_UMIN, Ty}))
3539 return lowerAddSubSatToMinMax(MI);
3541 }
3542 case G_SADDSAT:
3543 case G_SSUBSAT: {
3544 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3545
3546 // FIXME: It would probably make more sense to see if G_SADDO is preferred,
3547 // since it's a shorter expansion. However, we would need to figure out the
3548 // preferred boolean type for the carry out for the query.
3549 if (LI.isLegalOrCustom({G_SMIN, Ty}) && LI.isLegalOrCustom({G_SMAX, Ty}))
3550 return lowerAddSubSatToMinMax(MI);
3552 }
3553 case G_SSHLSAT:
3554 case G_USHLSAT:
3555 return lowerShlSat(MI);
3556 case G_ABS:
3557 return lowerAbsToAddXor(MI);
3558 case G_SELECT:
3559 return lowerSelect(MI);
3560 case G_IS_FPCLASS:
3561 return lowerISFPCLASS(MI);
3562 case G_SDIVREM:
3563 case G_UDIVREM:
3564 return lowerDIVREM(MI);
3565 case G_FSHL:
3566 case G_FSHR:
3567 return lowerFunnelShift(MI);
3568 case G_ROTL:
3569 case G_ROTR:
3570 return lowerRotate(MI);
3571 case G_MEMSET:
3572 case G_MEMCPY:
3573 case G_MEMMOVE:
3574 return lowerMemCpyFamily(MI);
3575 case G_MEMCPY_INLINE:
3576 return lowerMemcpyInline(MI);
3578 return lowerVectorReduction(MI);
3579 }
3580}
3581
3583 Align MinAlign) const {
3584 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
3585 // datalayout for the preferred alignment. Also there should be a target hook
3586 // for this to allow targets to reduce the alignment and ignore the
3587 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
3588 // the type.
3589 return std::max(Align(PowerOf2Ceil(Ty.getSizeInBytes())), MinAlign);
3590}
3591
3594 MachinePointerInfo &PtrInfo) {
3597 int FrameIdx = MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false);
3598
3599 unsigned AddrSpace = DL.getAllocaAddrSpace();
3600 LLT FramePtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
3601
3602 PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx);
3603 return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx);
3604}
3605
3607 LLT VecTy) {
3608 int64_t IdxVal;
3609 if (mi_match(IdxReg, *B.getMRI(), m_ICst(IdxVal)))
3610 return IdxReg;
3611
3612 LLT IdxTy = B.getMRI()->getType(IdxReg);
3613 unsigned NElts = VecTy.getNumElements();
3614 if (isPowerOf2_32(NElts)) {
3615 APInt Imm = APInt::getLowBitsSet(IdxTy.getSizeInBits(), Log2_32(NElts));
3616 return B.buildAnd(IdxTy, IdxReg, B.buildConstant(IdxTy, Imm)).getReg(0);
3617 }
3618
3619 return B.buildUMin(IdxTy, IdxReg, B.buildConstant(IdxTy, NElts - 1))
3620 .getReg(0);
3621}
3622
3624 Register Index) {
3625 LLT EltTy = VecTy.getElementType();
3626
3627 // Calculate the element offset and add it to the pointer.
3628 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
3629 assert(EltSize * 8 == EltTy.getSizeInBits() &&
3630 "Converting bits to bytes lost precision");
3631
3633
3634 LLT IdxTy = MRI.getType(Index);
3635 auto Mul = MIRBuilder.buildMul(IdxTy, Index,
3636 MIRBuilder.buildConstant(IdxTy, EltSize));
3637
3638 LLT PtrTy = MRI.getType(VecPtr);
3639 return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0);
3640}
3641
3642#ifndef NDEBUG
3643/// Check that all vector operands have same number of elements. Other operands
3644/// should be listed in NonVecOp.
3647 std::initializer_list<unsigned> NonVecOpIndices) {
3648 if (MI.getNumMemOperands() != 0)
3649 return false;
3650
3651 LLT VecTy = MRI.getType(MI.getReg(0));
3652 if (!VecTy.isVector())
3653 return false;
3654 unsigned NumElts = VecTy.getNumElements();
3655
3656 for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
3657 MachineOperand &Op = MI.getOperand(OpIdx);
3658 if (!Op.isReg()) {
3659 if (!is_contained(NonVecOpIndices, OpIdx))
3660 return false;
3661 continue;
3662 }
3663
3664 LLT Ty = MRI.getType(Op.getReg());
3665 if (!Ty.isVector()) {
3666 if (!is_contained(NonVecOpIndices, OpIdx))
3667 return false;
3668 continue;
3669 }
3670
3671 if (Ty.getNumElements() != NumElts)
3672 return false;
3673 }
3674
3675 return true;
3676}
3677#endif
3678
3679/// Fill \p DstOps with DstOps that have same number of elements combined as
3680/// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are
3681/// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple
3682/// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements.
3683static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty,
3684 unsigned NumElts) {
3685 LLT LeftoverTy;
3686 assert(Ty.isVector() && "Expected vector type");
3687 LLT EltTy = Ty.getElementType();
3688 LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy);
3689 int NumParts, NumLeftover;
3690 std::tie(NumParts, NumLeftover) =
3691 getNarrowTypeBreakDown(Ty, NarrowTy, LeftoverTy);
3692
3693 assert(NumParts > 0 && "Error in getNarrowTypeBreakDown");
3694 for (int i = 0; i < NumParts; ++i) {
3695 DstOps.push_back(NarrowTy);
3696 }
3697
3698 if (LeftoverTy.isValid()) {
3699 assert(NumLeftover == 1 && "expected exactly one leftover");
3700 DstOps.push_back(LeftoverTy);
3701 }
3702}
3703
3704/// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps
3705/// made from \p Op depending on operand type.
3706static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N,
3707 MachineOperand &Op) {
3708 for (unsigned i = 0; i < N; ++i) {
3709 if (Op.isReg())
3710 Ops.push_back(Op.getReg());
3711 else if (Op.isImm())
3712 Ops.push_back(Op.getImm());
3713 else if (Op.isPredicate())
3714 Ops.push_back(static_cast<CmpInst::Predicate>(Op.getPredicate()));
3715 else
3716 llvm_unreachable("Unsupported type");
3717 }
3718}
3719
3720// Handle splitting vector operations which need to have the same number of
3721// elements in each type index, but each type index may have a different element
3722// type.
3723//
3724// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
3725// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3726// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3727//
3728// Also handles some irregular breakdown cases, e.g.
3729// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
3730// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3731// s64 = G_SHL s64, s32
3734 GenericMachineInstr &MI, unsigned NumElts,
3735 std::initializer_list<unsigned> NonVecOpIndices) {
3736 assert(hasSameNumEltsOnAllVectorOperands(MI, MRI, NonVecOpIndices) &&
3737 "Non-compatible opcode or not specified non-vector operands");
3738 unsigned OrigNumElts = MRI.getType(MI.getReg(0)).getNumElements();
3739
3740 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
3741 unsigned NumDefs = MI.getNumDefs();
3742
3743 // Create DstOps (sub-vectors with NumElts elts + Leftover) for each output.
3744 // Build instructions with DstOps to use instruction found by CSE directly.
3745 // CSE copies found instruction into given vreg when building with vreg dest.
3746 SmallVector<SmallVector<DstOp, 8>, 2> OutputOpsPieces(NumDefs);
3747 // Output registers will be taken from created instructions.
3748 SmallVector<SmallVector<Register, 8>, 2> OutputRegs(NumDefs);
3749 for (unsigned i = 0; i < NumDefs; ++i) {
3750 makeDstOps(OutputOpsPieces[i], MRI.getType(MI.getReg(i)), NumElts);
3751 }
3752
3753 // Split vector input operands into sub-vectors with NumElts elts + Leftover.
3754 // Operands listed in NonVecOpIndices will be used as is without splitting;
3755 // examples: compare predicate in icmp and fcmp (op 1), vector select with i1
3756 // scalar condition (op 1), immediate in sext_inreg (op 2).
3757 SmallVector<SmallVector<SrcOp, 8>, 3> InputOpsPieces(NumInputs);
3758 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
3759 ++UseIdx, ++UseNo) {
3760 if (is_contained(NonVecOpIndices, UseIdx)) {
3761 broadcastSrcOp(InputOpsPieces[UseNo], OutputOpsPieces[0].size(),
3762 MI.getOperand(UseIdx));
3763 } else {
3764 SmallVector<Register, 8> SplitPieces;
3765 extractVectorParts(MI.getReg(UseIdx), NumElts, SplitPieces);
3766 for (auto Reg : SplitPieces)
3767 InputOpsPieces[UseNo].push_back(Reg);
3768 }
3769 }
3770
3771 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
3772
3773 // Take i-th piece of each input operand split and build sub-vector/scalar
3774 // instruction. Set i-th DstOp(s) from OutputOpsPieces as destination(s).
3775 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
3777 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
3778 Defs.push_back(OutputOpsPieces[DstNo][i]);
3779
3781 for (unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
3782 Uses.push_back(InputOpsPieces[InputNo][i]);
3783
3784 auto I = MIRBuilder.buildInstr(MI.getOpcode(), Defs, Uses, MI.getFlags());
3785 for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
3786 OutputRegs[DstNo].push_back(I.getReg(DstNo));
3787 }
3788
3789 // Merge small outputs into MI's output for each def operand.
3790 if (NumLeftovers) {
3791 for (unsigned i = 0; i < NumDefs; ++i)
3792 mergeMixedSubvectors(MI.getReg(i), OutputRegs[i]);
3793 } else {
3794 for (unsigned i = 0; i < NumDefs; ++i)
3795 MIRBuilder.buildMergeLikeInstr(MI.getReg(i), OutputRegs[i]);
3796 }
3797
3798 MI.eraseFromParent();
3799 return Legalized;
3800}
3801
3804 unsigned NumElts) {
3805 unsigned OrigNumElts = MRI.getType(MI.getReg(0)).getNumElements();
3806
3807 unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs();
3808 unsigned NumDefs = MI.getNumDefs();
3809
3810 SmallVector<DstOp, 8> OutputOpsPieces;
3811 SmallVector<Register, 8> OutputRegs;
3812 makeDstOps(OutputOpsPieces, MRI.getType(MI.getReg(0)), NumElts);
3813
3814 // Instructions that perform register split will be inserted in basic block
3815 // where register is defined (basic block is in the next operand).
3816 SmallVector<SmallVector<Register, 8>, 3> InputOpsPieces(NumInputs / 2);
3817 for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands();
3818 UseIdx += 2, ++UseNo) {
3819 MachineBasicBlock &OpMBB = *MI.getOperand(UseIdx + 1).getMBB();
3821 extractVectorParts(MI.getReg(UseIdx), NumElts, InputOpsPieces[UseNo]);
3822 }
3823
3824 // Build PHIs with fewer elements.
3825 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
3826 MIRBuilder.setInsertPt(*MI.getParent(), MI);
3827 for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
3828 auto Phi = MIRBuilder.buildInstr(TargetOpcode::G_PHI);
3829 Phi.addDef(
3830 MRI.createGenericVirtualRegister(OutputOpsPieces[i].getLLTTy(MRI)));
3831 OutputRegs.push_back(Phi.getReg(0));
3832
3833 for (unsigned j = 0; j < NumInputs / 2; ++j) {
3834 Phi.addUse(InputOpsPieces[j][i]);
3835 Phi.add(MI.getOperand(1 + j * 2 + 1));
3836 }
3837 }
3838
3839 // Merge small outputs into MI's def.
3840 if (NumLeftovers) {
3841 mergeMixedSubvectors(MI.getReg(0), OutputRegs);
3842 } else {
3843 MIRBuilder.buildMergeLikeInstr(MI.getReg(0), OutputRegs);
3844 }
3845
3846 MI.eraseFromParent();
3847 return Legalized;
3848}
3849
3852 unsigned TypeIdx,
3853 LLT NarrowTy) {
3854 const int NumDst = MI.getNumOperands() - 1;
3855 const Register SrcReg = MI.getOperand(NumDst).getReg();
3856 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3857 LLT SrcTy = MRI.getType(SrcReg);
3858
3859 if (TypeIdx != 1 || NarrowTy == DstTy)
3860 return UnableToLegalize;
3861
3862 // Requires compatible types. Otherwise SrcReg should have been defined by
3863 // merge-like instruction that would get artifact combined. Most likely
3864 // instruction that defines SrcReg has to perform more/fewer elements
3865 // legalization compatible with NarrowTy.
3866 assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types");
3867 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
3868
3869 if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
3870 (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0))
3871 return UnableToLegalize;
3872
3873 // This is most likely DstTy (smaller then register size) packed in SrcTy
3874 // (larger then register size) and since unmerge was not combined it will be
3875 // lowered to bit sequence extracts from register. Unpack SrcTy to NarrowTy
3876 // (register size) pieces first. Then unpack each of NarrowTy pieces to DstTy.
3877
3878 // %1:_(DstTy), %2, %3, %4 = G_UNMERGE_VALUES %0:_(SrcTy)
3879 //
3880 // %5:_(NarrowTy), %6 = G_UNMERGE_VALUES %0:_(SrcTy) - reg sequence
3881 // %1:_(DstTy), %2 = G_UNMERGE_VALUES %5:_(NarrowTy) - sequence of bits in reg
3882 // %3:_(DstTy), %4 = G_UNMERGE_VALUES %6:_(NarrowTy)
3883 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, SrcReg);
3884 const int NumUnmerge = Unmerge->getNumOperands() - 1;
3885 const int PartsPerUnmerge = NumDst / NumUnmerge;
3886
3887 for (int I = 0; I != NumUnmerge; ++I) {
3888 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
3889
3890 for (int J = 0; J != PartsPerUnmerge; ++J)
3891 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg());
3892 MIB.addUse(Unmerge.getReg(I));
3893 }
3894
3895 MI.eraseFromParent();
3896 return Legalized;
3897}
3898
3901 LLT NarrowTy) {
3902 Register DstReg = MI.getOperand(0).getReg();
3903 LLT DstTy = MRI.getType(DstReg);
3904 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
3905 // Requires compatible types. Otherwise user of DstReg did not perform unmerge
3906 // that should have been artifact combined. Most likely instruction that uses
3907 // DstReg has to do more/fewer elements legalization compatible with NarrowTy.
3908 assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types");
3909 assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
3910 if (NarrowTy == SrcTy)
3911 return UnableToLegalize;
3912
3913 // This attempts to lower part of LCMTy merge/unmerge sequence. Intended use
3914 // is for old mir tests. Since the changes to more/fewer elements it should no
3915 // longer be possible to generate MIR like this when starting from llvm-ir
3916 // because LCMTy approach was replaced with merge/unmerge to vector elements.
3917 if (TypeIdx == 1) {
3918 assert(SrcTy.isVector() && "Expected vector types");
3919 assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type");
3920 if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) ||
3921 (NarrowTy.getNumElements() >= SrcTy.getNumElements()))
3922 return UnableToLegalize;
3923 // %2:_(DstTy) = G_CONCAT_VECTORS %0:_(SrcTy), %1:_(SrcTy)
3924 //
3925 // %3:_(EltTy), %4, %5 = G_UNMERGE_VALUES %0:_(SrcTy)
3926 // %6:_(EltTy), %7, %8 = G_UNMERGE_VALUES %1:_(SrcTy)
3927 // %9:_(NarrowTy) = G_BUILD_VECTOR %3:_(EltTy), %4
3928 // %10:_(NarrowTy) = G_BUILD_VECTOR %5:_(EltTy), %6
3929 // %11:_(NarrowTy) = G_BUILD_VECTOR %7:_(EltTy), %8
3930 // %2:_(DstTy) = G_CONCAT_VECTORS %9:_(NarrowTy), %10, %11
3931
3933 LLT EltTy = MRI.getType(MI.getOperand(1).getReg()).getScalarType();
3934 for (unsigned i = 1; i < MI.getNumOperands(); ++i) {
3935 auto Unmerge = MIRBuilder.buildUnmerge(EltTy, MI.getOperand(i).getReg());
3936 for (unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
3937 Elts.push_back(Unmerge.getReg(j));
3938 }
3939
3940 SmallVector<Register, 8> NarrowTyElts;
3941 unsigned NumNarrowTyElts = NarrowTy.getNumElements();
3942 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
3943 for (unsigned i = 0, Offset = 0; i < NumNarrowTyPieces;
3944 ++i, Offset += NumNarrowTyElts) {
3945 ArrayRef<Register> Pieces(&Elts[Offset], NumNarrowTyElts);
3946 NarrowTyElts.push_back(
3947 MIRBuilder.buildMergeLikeInstr(NarrowTy, Pieces).getReg(0));
3948 }
3949
3950 MIRBuilder.buildMergeLikeInstr(DstReg, NarrowTyElts);
3951 MI.eraseFromParent();
3952 return Legalized;
3953 }
3954
3955 assert(TypeIdx == 0 && "Bad type index");
3956 if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) ||
3957 (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0))
3958 return UnableToLegalize;
3959
3960 // This is most likely SrcTy (smaller then register size) packed in DstTy
3961 // (larger then register size) and since merge was not combined it will be
3962 // lowered to bit sequence packing into register. Merge SrcTy to NarrowTy
3963 // (register size) pieces first. Then merge each of NarrowTy pieces to DstTy.
3964
3965 // %0:_(DstTy) = G_MERGE_VALUES %1:_(SrcTy), %2, %3, %4
3966 //
3967 // %5:_(NarrowTy) = G_MERGE_VALUES %1:_(SrcTy), %2 - sequence of bits in reg
3968 // %6:_(NarrowTy) = G_MERGE_VALUES %3:_(SrcTy), %4
3969 // %0:_(DstTy) = G_MERGE_VALUES %5:_(NarrowTy), %6 - reg sequence
3970 SmallVector<Register, 8> NarrowTyElts;
3971 unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
3972 unsigned NumSrcElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
3973 unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts;
3974 for (unsigned i = 0; i < NumParts; ++i) {
3976 for (unsigned j = 0; j < NumElts; ++j)
3977 Sources.push_back(MI.getOperand(1 + i * NumElts + j).getReg());
3978 NarrowTyElts.push_back(
3979 MIRBuilder.buildMergeLikeInstr(NarrowTy, Sources).getReg(0));
3980 }
3981
3982 MIRBuilder.buildMergeLikeInstr(DstReg, NarrowTyElts);
3983 MI.eraseFromParent();
3984 return Legalized;
3985}
3986
3989 unsigned TypeIdx,
3990 LLT NarrowVecTy) {
3991 Register DstReg = MI.getOperand(0).getReg();
3992 Register SrcVec = MI.getOperand(1).getReg();
3993 Register InsertVal;
3994 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
3995
3996 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index");
3997 if (IsInsert)
3998 InsertVal = MI.getOperand(2).getReg();
3999
4000 Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg();
4001
4002 // TODO: Handle total scalarization case.
4003 if (!NarrowVecTy.isVector())
4004 return UnableToLegalize;
4005
4006 LLT VecTy = MRI.getType(SrcVec);
4007
4008 // If the index is a constant, we can really break this down as you would
4009 // expect, and index into the target size pieces.
4010 int64_t IdxVal;
4011 auto MaybeCst = getIConstantVRegValWithLookThrough(Idx, MRI);
4012 if (MaybeCst) {
4013 IdxVal = MaybeCst->Value.getSExtValue();
4014 // Avoid out of bounds indexing the pieces.
4015 if (IdxVal >= VecTy.getNumElements()) {
4016 MIRBuilder.buildUndef(DstReg);
4017 MI.eraseFromParent();
4018 return Legalized;
4019 }
4020
4021 SmallVector<Register, 8> VecParts;
4022 LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec);
4023
4024 // Build a sequence of NarrowTy pieces in VecParts for this operand.
4025 LLT LCMTy = buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts,
4026 TargetOpcode::G_ANYEXT);
4027
4028 unsigned NewNumElts = NarrowVecTy.getNumElements();
4029
4030 LLT IdxTy = MRI.getType(Idx);
4031 int64_t PartIdx = IdxVal / NewNumElts;
4032 auto NewIdx =
4033 MIRBuilder.buildConstant(IdxTy, IdxVal - NewNumElts * PartIdx);
4034
4035 if (IsInsert) {
4036 LLT PartTy = MRI.getType(VecParts[PartIdx]);
4037
4038 // Use the adjusted index to insert into one of the subvectors.
4039 auto InsertPart = MIRBuilder.buildInsertVectorElement(
4040 PartTy, VecParts[PartIdx], InsertVal, NewIdx);
4041 VecParts[PartIdx] = InsertPart.getReg(0);
4042
4043 // Recombine the inserted subvector with the others to reform the result
4044 // vector.
4045 buildWidenedRemergeToDst(DstReg, LCMTy, VecParts);
4046 } else {
4047 MIRBuilder.buildExtractVectorElement(DstReg, VecParts[PartIdx], NewIdx);
4048 }
4049
4050 MI.eraseFromParent();
4051 return Legalized;
4052 }
4053
4054 // With a variable index, we can't perform the operation in a smaller type, so
4055 // we're forced to expand this.
4056 //
4057 // TODO: We could emit a chain of compare/select to figure out which piece to
4058 // index.
4060}
4061
4064 LLT NarrowTy) {
4065 // FIXME: Don't know how to handle secondary types yet.
4066 if (TypeIdx != 0)
4067 return UnableToLegalize;
4068
4069 // This implementation doesn't work for atomics. Give up instead of doing
4070 // something invalid.
4071 if (LdStMI.isAtomic())
4072 return UnableToLegalize;
4073
4074 bool IsLoad = isa<GLoad>(LdStMI);
4075 Register ValReg = LdStMI.getReg(0);
4076 Register AddrReg = LdStMI.getPointerReg();
4077 LLT ValTy = MRI.getType(ValReg);
4078
4079 // FIXME: Do we need a distinct NarrowMemory legalize action?
4080 if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize()) {
4081 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
4082 return UnableToLegalize;
4083 }
4084
4085 int NumParts = -1;
4086 int NumLeftover = -1;
4087 LLT LeftoverTy;
4088 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
4089 if (IsLoad) {
4090 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
4091 } else {
4092 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
4093 NarrowLeftoverRegs)) {
4094 NumParts = NarrowRegs.size();
4095 NumLeftover = NarrowLeftoverRegs.size();
4096 }
4097 }
4098
4099 if (NumParts == -1)
4100 return UnableToLegalize;
4101
4102 LLT PtrTy = MRI.getType(AddrReg);
4103 const LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits());
4104
4105 unsigned TotalSize = ValTy.getSizeInBits();
4106
4107 // Split the load/store into PartTy sized pieces starting at Offset. If this
4108 // is a load, return the new registers in ValRegs. For a store, each elements
4109 // of ValRegs should be PartTy. Returns the next offset that needs to be
4110 // handled.
4112 auto MMO = LdStMI.getMMO();
4113 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
4114 unsigned NumParts, unsigned Offset) -> unsigned {
4116 unsigned PartSize = PartTy.getSizeInBits();
4117 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
4118 ++Idx) {
4119 unsigned ByteOffset = Offset / 8;
4120 Register NewAddrReg;
4121
4122 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
4123
4124 MachineMemOperand *NewMMO =
4125 MF.getMachineMemOperand(&MMO, ByteOffset, PartTy);
4126
4127 if (IsLoad) {
4128 Register Dst = MRI.createGenericVirtualRegister(PartTy);
4129 ValRegs.push_back(Dst);
4130 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
4131 } else {
4132 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
4133 }
4134 Offset = isBigEndian ? Offset - PartSize : Offset + PartSize;
4135 }
4136
4137 return Offset;
4138 };
4139
4140 unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0;
4141 unsigned HandledOffset =
4142 splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset);
4143
4144 // Handle the rest of the register if this isn't an even type breakdown.
4145 if (LeftoverTy.isValid())
4146 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
4147
4148 if (IsLoad) {
4149 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
4150 LeftoverTy, NarrowLeftoverRegs);
4151 }
4152
4153 LdStMI.eraseFromParent();
4154 return Legalized;
4155}
4156
4159 LLT NarrowTy) {
4160 using namespace TargetOpcode;
4161 GenericMachineInstr &GMI = cast<GenericMachineInstr>(MI);
4162 unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
4163
4164 switch (MI.getOpcode()) {
4165 case G_IMPLICIT_DEF:
4166 case G_TRUNC:
4167 case G_AND:
4168 case G_OR:
4169 case G_XOR:
4170 case G_ADD:
4171 case G_SUB:
4172 case G_MUL:
4173 case G_PTR_ADD:
4174 case G_SMULH:
4175 case G_UMULH:
4176 case G_FADD:
4177 case G_FMUL:
4178 case G_FSUB:
4179 case G_FNEG:
4180 case G_FABS:
4181 case G_FCANONICALIZE:
4182 case G_FDIV:
4183 case G_FREM:
4184 case G_FMA:
4185 case G_FMAD:
4186 case G_FPOW:
4187 case G_FEXP:
4188 case G_FEXP2:
4189 case G_FLOG:
4190 case G_FLOG2:
4191 case G_FLOG10:
4192 case G_FNEARBYINT:
4193 case G_FCEIL:
4194 case G_FFLOOR:
4195 case G_FRINT:
4196 case G_INTRINSIC_ROUND:
4197 case G_INTRINSIC_ROUNDEVEN:
4198 case G_INTRINSIC_TRUNC:
4199 case G_FCOS:
4200 case G_FSIN:
4201 case G_FSQRT:
4202 case G_BSWAP:
4203 case G_BITREVERSE:
4204 case G_SDIV:
4205 case G_UDIV:
4206 case G_SREM:
4207 case G_UREM:
4208 case G_SDIVREM:
4209 case G_UDIVREM:
4210 case G_SMIN:
4211 case G_SMAX:
4212 case G_UMIN:
4213 case G_UMAX:
4214 case G_ABS:
4215 case G_FMINNUM:
4216 case G_FMAXNUM:
4217 case G_FMINNUM_IEEE:
4218 case G_FMAXNUM_IEEE:
4219 case G_FMINIMUM:
4220 case G_FMAXIMUM:
4221 case G_FSHL:
4222 case G_FSHR:
4223 case G_ROTL:
4224 case G_ROTR:
4225 case G_FREEZE:
4226 case G_SADDSAT:
4227 case G_SSUBSAT:
4228 case G_UADDSAT:
4229 case G_USUBSAT:
4230 case G_UMULO:
4231 case G_SMULO:
4232 case G_SHL:
4233 case G_LSHR:
4234 case G_ASHR:
4235 case G_SSHLSAT:
4236 case G_USHLSAT:
4237 case G_CTLZ:
4238 case G_CTLZ_ZERO_UNDEF:
4239 case G_CTTZ:
4240 case G_CTTZ_ZERO_UNDEF:
4241 case G_CTPOP:
4242 case G_FCOPYSIGN:
4243 case G_ZEXT:
4244 case G_SEXT:
4245 case G_ANYEXT:
4246 case G_FPEXT:
4247 case G_FPTRUNC:
4248 case G_SITOFP:
4249 case G_UITOFP:
4250 case G_FPTOSI:
4251 case G_FPTOUI:
4252 case G_INTTOPTR:
4253 case G_PTRTOINT:
4254 case G_ADDRSPACE_CAST:
4255 case G_UADDO:
4256 case G_USUBO:
4257 case G_UADDE:
4258 case G_USUBE:
4259 case G_SADDO:
4260 case G_SSUBO:
4261 case G_SADDE:
4262 case G_SSUBE:
4263 case G_STRICT_FADD:
4264 case G_STRICT_FSUB:
4265 case G_STRICT_FMUL:
4266 case G_STRICT_FMA:
4267 return fewerElementsVectorMultiEltType(GMI, NumElts);
4268 case G_ICMP:
4269 case G_FCMP:
4270 return fewerElementsVectorMultiEltType(GMI, NumElts, {1 /*cpm predicate*/});
4271 case G_IS_FPCLASS:
4272 return fewerElementsVectorMultiEltType(GMI, NumElts, {2, 3 /*mask,fpsem*/});
4273 case G_SELECT:
4274 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
4275 return fewerElementsVectorMultiEltType(GMI, NumElts);
4276 return fewerElementsVectorMultiEltType(GMI, NumElts, {1 /*scalar cond*/});
4277 case G_PHI:
4278 return fewerElementsVectorPhi(GMI, NumElts);
4279 case G_UNMERGE_VALUES:
4280 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
4281 case G_BUILD_VECTOR:
4282 assert(TypeIdx == 0 && "not a vector type index");
4283 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
4284 case G_CONCAT_VECTORS:
4285 if (TypeIdx != 1) // TODO: This probably does work as expected already.
4286 return UnableToLegalize;
4287 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
4288 case G_EXTRACT_VECTOR_ELT:
4289 case G_INSERT_VECTOR_ELT:
4290 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowTy);
4291 case G_LOAD:
4292 case G_STORE:
4293 return reduceLoadStoreWidth(cast<GLoadStore>(MI), TypeIdx, NarrowTy);
4294 case G_SEXT_INREG:
4295 return fewerElementsVectorMultiEltType(GMI, NumElts, {2 /*imm*/});
4297 return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy);
4298 case G_SHUFFLE_VECTOR:
4299 return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy);
4300 default:
4301 return UnableToLegalize;
4302 }
4303}
4304
4306 MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) {
4307 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
4308 if (TypeIdx != 0)
4309 return UnableToLegalize;
4310
4311 Register DstReg = MI.getOperand(0).getReg();
4312 Register Src1Reg = MI.getOperand(1).getReg();
4313 Register Src2Reg = MI.getOperand(2).getReg();
4314 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
4315 LLT DstTy = MRI.getType(DstReg);
4316 LLT Src1Ty = MRI.getType(Src1Reg);
4317 LLT Src2Ty = MRI.getType(Src2Reg);
4318 // The shuffle should be canonicalized by now.
4319 if (DstTy != Src1Ty)
4320 return UnableToLegalize;
4321 if (DstTy != Src2Ty)
4322 return UnableToLegalize;
4323
4324 if (!isPowerOf2_32(DstTy.getNumElements()))
4325 return UnableToLegalize;
4326
4327 // We only support splitting a shuffle into 2, so adjust NarrowTy accordingly.
4328 // Further legalization attempts will be needed to do split further.
4329 NarrowTy =
4331 unsigned NewElts = NarrowTy.getNumElements();
4332
4333 SmallVector<Register> SplitSrc1Regs, SplitSrc2Regs;
4334 extractParts(Src1Reg, NarrowTy, 2, SplitSrc1Regs);
4335 extractParts(Src2Reg, NarrowTy, 2, SplitSrc2Regs);
4336 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
4337 SplitSrc2Regs[1]};
4338
4339 Register Hi, Lo;
4340
4341 // If Lo or Hi uses elements from at most two of the four input vectors, then
4342 // express it as a vector shuffle of those two inputs. Otherwise extract the
4343 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
4345 for (unsigned High = 0; High < 2; ++High) {
4346 Register &Output = High ? Hi : Lo;
4347
4348 // Build a shuffle mask for the output, discovering on the fly which
4349 // input vectors to use as shuffle operands (recorded in InputUsed).
4350 // If building a suitable shuffle vector proves too hard, then bail
4351 // out with useBuildVector set.
4352 unsigned InputUsed[2] = {-1U, -1U}; // Not yet discovered.
4353 unsigned FirstMaskIdx = High * NewElts;
4354 bool UseBuildVector = false;
4355 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4356 // The mask element. This indexes into the input.
4357 int Idx = Mask[FirstMaskIdx + MaskOffset];
4358
4359 // The input vector this mask element indexes into.
4360 unsigned Input = (unsigned)Idx / NewElts;
4361
4362 if (Input >= std::size(Inputs)) {
4363 // The mask element does not index into any input vector.
4364 Ops.push_back(-1);
4365 continue;
4366 }
4367
4368 // Turn the index into an offset from the start of the input vector.
4369 Idx -= Input * NewElts;
4370
4371 // Find or create a shuffle vector operand to hold this input.
4372 unsigned OpNo;
4373 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
4374 if (InputUsed[OpNo] == Input) {
4375 // This input vector is already an operand.
4376 break;
4377 } else if (InputUsed[OpNo] == -1U) {
4378 // Create a new operand for this input vector.
4379 InputUsed[OpNo] = Input;
4380 break;
4381 }
4382 }
4383
4384 if (OpNo >= std::size(InputUsed)) {
4385 // More than two input vectors used! Give up on trying to create a
4386 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
4387 UseBuildVector = true;
4388 break;
4389 }
4390
4391 // Add the mask index for the new shuffle vector.
4392 Ops.push_back(Idx + OpNo * NewElts);
4393 }
4394
4395 if (UseBuildVector) {
4396 LLT EltTy = NarrowTy.getElementType();
4398
4399 // Extract the input elements by hand.
4400 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4401 // The mask element. This indexes into the input.
4402 int Idx = Mask[FirstMaskIdx + MaskOffset];
4403
4404 // The input vector this mask element indexes into.
4405 unsigned Input = (unsigned)Idx / NewElts;
4406
4407 if (Input >= std::size(Inputs)) {
4408 // The mask element is "undef" or indexes off the end of the input.
4409 SVOps.push_back(MIRBuilder.buildUndef(EltTy).getReg(0));
4410 continue;
4411 }
4412
4413 // Turn the index into an offset from the start of the input vector.
4414 Idx -= Input * NewElts;
4415
4416 // Extract the vector element by hand.
4417 SVOps.push_back(MIRBuilder
4418 .buildExtractVectorElement(
4419 EltTy, Inputs[Input],
4421 .getReg(0));
4422 }
4423
4424 // Construct the Lo/Hi output using a G_BUILD_VECTOR.
4425 Output = MIRBuilder.buildBuildVector(NarrowTy, SVOps).getReg(0);
4426 } else if (InputUsed[0] == -1U) {
4427 // No input vectors were used! The result is undefined.
4428 Output = MIRBuilder.buildUndef(NarrowTy).getReg(0);
4429 } else {
4430 Register Op0 = Inputs[InputUsed[0]];
4431 // If only one input was used, use an undefined vector for the other.
4432 Register Op1 = InputUsed[1] == -1U
4433 ? MIRBuilder.buildUndef(NarrowTy).getReg(0)
4434 : Inputs[InputUsed[1]];
4435 // At least one input vector was used. Create a new shuffle vector.
4436 Output = MIRBuilder.buildShuffleVector(NarrowTy, Op0, Op1, Ops).getReg(0);
4437 }
4438
4439 Ops.clear();
4440 }
4441
4442 MIRBuilder.buildConcatVectors(DstReg, {Lo, Hi});
4443 MI.eraseFromParent();
4444 return Legalized;
4445}
4446
4447static unsigned getScalarOpcForReduction(unsigned Opc) {
4448 unsigned ScalarOpc;
4449 switch (Opc) {
4450 case TargetOpcode::G_VECREDUCE_FADD:
4451 ScalarOpc = TargetOpcode::G_FADD;
4452 break;
4453 case TargetOpcode::G_VECREDUCE_FMUL:
4454 ScalarOpc = TargetOpcode::G_FMUL;
4455 break;
4456 case TargetOpcode::G_VECREDUCE_FMAX:
4457 ScalarOpc = TargetOpcode::G_FMAXNUM;
4458 break;
4459 case TargetOpcode::G_VECREDUCE_FMIN:
4460 ScalarOpc = TargetOpcode::G_FMINNUM;
4461 break;
4462 case TargetOpcode::G_VECREDUCE_ADD:
4463 ScalarOpc = TargetOpcode::G_ADD;
4464 break;
4465 case TargetOpcode::G_VECREDUCE_MUL:
4466 ScalarOpc = TargetOpcode::G_MUL;
4467 break;
4468 case TargetOpcode::G_VECREDUCE_AND:
4469 ScalarOpc = TargetOpcode::G_AND;
4470 break;
4471 case TargetOpcode::G_VECREDUCE_OR: